flacfetch 0.36.0__tar.gz → 0.37.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. {flacfetch-0.36.0/flacfetch.egg-info → flacfetch-0.37.1}/PKG-INFO +1 -1
  2. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/search.py +13 -2
  3. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/interfaces.py +4 -0
  4. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/manager.py +6 -0
  5. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/keeper.py +181 -65
  6. flacfetch-0.37.1/flacfetch/credential_keeper/refresh_request.py +113 -0
  7. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/youtube.py +11 -0
  8. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/downloaders/spotify.py +39 -4
  9. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/downloaders/youtube.py +33 -12
  10. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/providers/youtube.py +67 -5
  11. {flacfetch-0.36.0 → flacfetch-0.37.1/flacfetch.egg-info}/PKG-INFO +1 -1
  12. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch.egg-info/SOURCES.txt +3 -0
  13. {flacfetch-0.36.0 → flacfetch-0.37.1}/pyproject.toml +1 -1
  14. flacfetch-0.37.1/tests/test_cookie_refresh_request.py +108 -0
  15. flacfetch-0.37.1/tests/test_keeper_leaves_youtube.py +43 -0
  16. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_keeper_probe.py +242 -0
  17. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_manager.py +41 -0
  18. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_search_api.py +37 -0
  19. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_spotify.py +83 -0
  20. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_youtube.py +117 -0
  21. flacfetch-0.37.1/tests/test_youtube_provider.py +167 -0
  22. flacfetch-0.36.0/tests/test_youtube_provider.py +0 -32
  23. {flacfetch-0.36.0 → flacfetch-0.37.1}/ARCHITECTURE.md +0 -0
  24. {flacfetch-0.36.0 → flacfetch-0.37.1}/LICENSE +0 -0
  25. {flacfetch-0.36.0 → flacfetch-0.37.1}/MANIFEST.in +0 -0
  26. {flacfetch-0.36.0 → flacfetch-0.37.1}/PLAN.md +0 -0
  27. {flacfetch-0.36.0 → flacfetch-0.37.1}/README.md +0 -0
  28. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/__init__.py +0 -0
  29. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/__init__.py +0 -0
  30. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/auth.py +0 -0
  31. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/main.py +0 -0
  32. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/models.py +0 -0
  33. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/__init__.py +0 -0
  34. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/cache.py +0 -0
  35. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/config.py +0 -0
  36. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/credentials.py +0 -0
  37. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/download.py +0 -0
  38. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/health.py +0 -0
  39. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/spotify.py +0 -0
  40. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/routes/torrents.py +0 -0
  41. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/__init__.py +0 -0
  42. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/credential_check.py +0 -0
  43. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/disk_manager.py +0 -0
  44. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/download_manager.py +0 -0
  45. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/health_check.py +0 -0
  46. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/search_cache.py +0 -0
  47. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/secret_manager_utils.py +0 -0
  48. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/api/services/spotify_budget.py +0 -0
  49. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/album_match.py +0 -0
  50. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/categorize.py +0 -0
  51. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/config.py +0 -0
  52. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/log.py +0 -0
  53. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/matching.py +0 -0
  54. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/core/models.py +0 -0
  55. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/__init__.py +0 -0
  56. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/__main__.py +0 -0
  57. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/browser.py +0 -0
  58. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/google_login.py +0 -0
  59. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/librespot.py +0 -0
  60. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/probe.py +0 -0
  61. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/credential_keeper/spotify.py +0 -0
  62. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/downloaders/torrent.py +0 -0
  63. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/downloaders/torrent_coordinator.py +0 -0
  64. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/interface/cli.py +0 -0
  65. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/interface/cli_remote.py +0 -0
  66. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/providers/gazelle.py +0 -0
  67. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/providers/ops.py +0 -0
  68. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/providers/red.py +0 -0
  69. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch/providers/spotify.py +0 -0
  70. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch.egg-info/dependency_links.txt +0 -0
  71. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch.egg-info/entry_points.txt +0 -0
  72. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch.egg-info/requires.txt +0 -0
  73. {flacfetch-0.36.0 → flacfetch-0.37.1}/flacfetch.egg-info/top_level.txt +0 -0
  74. {flacfetch-0.36.0 → flacfetch-0.37.1}/requirements.txt +0 -0
  75. {flacfetch-0.36.0 → flacfetch-0.37.1}/setup.cfg +0 -0
  76. {flacfetch-0.36.0 → flacfetch-0.37.1}/setup.py +0 -0
  77. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/_route_paths.py +0 -0
  78. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_album_search.py +0 -0
  79. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_auth.py +0 -0
  80. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_config.py +0 -0
  81. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_deep_health.py +0 -0
  82. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_health_ytdlp.py +0 -0
  83. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_integration.py +0 -0
  84. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_models.py +0 -0
  85. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_routes.py +0 -0
  86. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_api_services.py +0 -0
  87. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_cache_api.py +0 -0
  88. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_cli_cookies.py +0 -0
  89. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_cli_remote.py +0 -0
  90. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_core.py +0 -0
  91. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_credential_keeper.py +0 -0
  92. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_gazelle.py +0 -0
  93. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_logging.py +0 -0
  94. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_matching.py +0 -0
  95. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_models.py +0 -0
  96. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_ops.py +0 -0
  97. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_provider_errors.py +0 -0
  98. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_red.py +0 -0
  99. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_spotify_budget.py +0 -0
  100. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_torrent_coordinator.py +0 -0
  101. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_torrent_downloader.py +0 -0
  102. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_torrent_shared_album.py +0 -0
  103. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_transcode_hf_noise.py +0 -0
  104. {flacfetch-0.36.0 → flacfetch-0.37.1}/tests/test_youtube_cookies.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flacfetch
3
- Version: 0.36.0
3
+ Version: 0.37.1
4
4
  Summary: Search and download high-quality audio from multiple sources
5
5
  Home-page: https://github.com/yourusername/flacfetch
6
6
  Author: Your Name
@@ -85,16 +85,27 @@ def _configure_and_search(fetch_manager, artist: str, title: str, exhaustive: bo
85
85
 
86
86
  from flacfetch.core.models import TrackQuery
87
87
  query = TrackQuery(artist=artist, title=title)
88
- return fetch_manager.search(query)
88
+ releases = fetch_manager.search(query)
89
+ # Read here, while no other sweep can overwrite it.
90
+ return releases, dict(fetch_manager.last_search_errors)
89
91
 
90
92
 
91
93
  async def _search_and_cache(fetch_manager, cache_service, artist: str, title: str, exhaustive: bool):
92
94
  """Run a provider sweep off the event loop and cache the results."""
93
95
  loop = asyncio.get_running_loop()
94
- releases = await loop.run_in_executor(
96
+ releases, errors = await loop.run_in_executor(
95
97
  _search_executor, _configure_and_search, fetch_manager, artist, title, exhaustive
96
98
  )
97
99
 
100
+ # A provider failing doesn't mean it has nothing: caching this sweep would hide
101
+ # that provider's results for the whole TTL, long after it recovers.
102
+ if errors:
103
+ logger.warning(
104
+ f"Not caching search for {artist} - {title}: "
105
+ f"{', '.join(sorted(errors))} failed"
106
+ )
107
+ return releases
108
+
98
109
  # Cache results (fire-and-forget, best-effort)
99
110
  # Always cache fresh search results, including exhaustive searches
100
111
  if releases:
@@ -5,6 +5,10 @@ from .models import Release, TrackQuery
5
5
 
6
6
 
7
7
  class Provider(ABC):
8
+ # Why the latest search() is incomplete, if it swallowed a failure. None means
9
+ # the results (even if empty) can be trusted and cached.
10
+ last_search_error: Optional[str] = None
11
+
8
12
  @property
9
13
  @abstractmethod
10
14
  def name(self) -> str:
@@ -21,6 +21,8 @@ class FetchManager:
21
21
  self._default_downloader: Optional[Downloader] = None
22
22
  self._provider_priority: Optional[list[str]] = None
23
23
  self._use_fallback_search: bool = True # Search lower priority providers if higher ones fail
24
+ # Providers that failed during the latest search(): {name: reason}.
25
+ self.last_search_errors: dict[str, str] = {}
24
26
 
25
27
  def add_provider(self, provider: Provider):
26
28
  self.providers.append(provider)
@@ -51,6 +53,7 @@ class FetchManager:
51
53
 
52
54
  def search(self, query: TrackQuery) -> list[Release]:
53
55
  all_releases = []
56
+ self.last_search_errors = {}
54
57
 
55
58
  # Get providers in priority order
56
59
  providers = self._get_ordered_providers()
@@ -60,6 +63,8 @@ class FetchManager:
60
63
  logger.info(f"Searching {provider.name} for '{query.artist} - {query.title}'...")
61
64
  results = provider.search(query)
62
65
  logger.info(f"Found {len(results)} results from {provider.name}")
66
+ if provider.last_search_error:
67
+ self.last_search_errors[provider.name] = provider.last_search_error
63
68
 
64
69
  if results:
65
70
  all_releases.extend(results)
@@ -78,6 +83,7 @@ class FetchManager:
78
83
 
79
84
  except Exception as e:
80
85
  logger.error(f"Error searching {provider.name}: {e}")
86
+ self.last_search_errors[provider.name] = str(e)
81
87
  # Continue to next provider on error if fallback enabled
82
88
  if self._use_fallback_search:
83
89
  logger.info(f"Provider {provider.name} failed, trying next provider...")
@@ -8,6 +8,7 @@ import json
8
8
  import logging
9
9
  import os
10
10
  import sys
11
+ from dataclasses import dataclass
11
12
  from datetime import datetime, timezone
12
13
  from pathlib import Path
13
14
 
@@ -16,12 +17,18 @@ from .browser import close_browser, launch_browser
16
17
  from .google_login import ensure_google_logged_in
17
18
  from .librespot import refresh_librespot_credentials
18
19
  from .probe import ProbeOutcome, probe_cookies
20
+ from .refresh_request import COOLDOWN_SECONDS as YOUTUBE_ON_DEMAND_COOLDOWN
21
+ from .refresh_request import consume_refresh_request, refresh_requested
19
22
  from .spotify import refresh_spotify_token
20
23
  from .youtube import refresh_youtube_cookies
21
24
 
22
25
  logger = logging.getLogger(__name__)
23
26
 
24
27
  # Refresh intervals (in seconds)
28
+ # Only applies while probes are inconclusive (for example the cookie file is
29
+ # missing). A passing probe never re-exports: the export is working, and the old
30
+ # unconditional 8h re-export put the browser back on youtube.com, where the new
31
+ # export died within 30 minutes (0 of 41 survived, 2026-10-05 to 11-03).
25
32
  YOUTUBE_REFRESH_INTERVAL = int(os.environ.get("KEEPER_YOUTUBE_INTERVAL", 8 * 3600)) # 8 hours
26
33
  # How often to validate the exported cookies with a real yt-dlp probe between
27
34
  # refreshes. YouTube can kill the exported session server-side within hours of
@@ -38,6 +45,10 @@ YOUTUBE_MAX_REFRESH_ATTEMPTS = int(os.environ.get("KEEPER_YOUTUBE_MAX_ATTEMPTS",
38
45
  YOUTUBE_FAILURE_ALERT_INTERVAL = int(
39
46
  os.environ.get("KEEPER_YOUTUBE_FAILURE_ALERT_INTERVAL", 4 * 3600) # 4 hours
40
47
  )
48
+ # More dead probes than this in a UTC day sends one notification. Self-heal hides
49
+ # these, but cookies are dead for up to a probe interval each time, and real
50
+ # searches and downloads fail during that gap.
51
+ YOUTUBE_CHURN_ALERT_THRESHOLD = int(os.environ.get("KEEPER_YOUTUBE_CHURN_THRESHOLD", 6))
41
52
  SPOTIFY_REFRESH_INTERVAL = int(os.environ.get("KEEPER_SPOTIFY_INTERVAL", 12 * 3600)) # 12 hours
42
53
  # librespot stored credentials are long-lived (no ~1h access-token expiry), so
43
54
  # this only needs to run occasionally to re-mint if they are ever invalidated.
@@ -87,6 +98,34 @@ def _save_status(status: dict):
87
98
  logger.warning(f"Could not save status file: {e}")
88
99
 
89
100
 
101
+ def _today() -> str:
102
+ return datetime.now(timezone.utc).date().isoformat()
103
+
104
+
105
+ def _record_youtube_event(status: dict, event: str) -> dict:
106
+ """Count a probe event in today's stats, logging yesterday's on rollover."""
107
+ yt_status = status.setdefault("youtube", {})
108
+ stats = yt_status.get("daily_stats")
109
+ today = _today()
110
+ if not stats or stats.get("date") != today:
111
+ if stats:
112
+ logger.info(
113
+ f"YouTube probe summary date={stats['date']} ok={stats['ok_probes']} "
114
+ f"dead={stats['dead_probes']} rejected={stats['rejected_exports']} "
115
+ f"on_demand={stats['on_demand_refreshes']}"
116
+ )
117
+ stats = yt_status["daily_stats"] = {
118
+ "date": today,
119
+ "ok_probes": 0,
120
+ "dead_probes": 0,
121
+ "rejected_exports": 0,
122
+ "on_demand_refreshes": 0,
123
+ "churn_alerted": False,
124
+ }
125
+ stats[event] += 1
126
+ return stats
127
+
128
+
90
129
  async def _send_notification(title: str, body: str):
91
130
  """Send a Pushbullet notification."""
92
131
  api_key = os.environ.get("PUSHBULLET_API_KEY")
@@ -172,6 +211,7 @@ async def refresh_and_validate_youtube(
172
211
 
173
212
  if outcome is ProbeOutcome.INVALID_COOKIES:
174
213
  failure_reason = msg
214
+ _record_youtube_event(status, "rejected_exports")
175
215
  logger.warning(
176
216
  f"Exported cookies rejected by probe (attempt {attempt}/{max_attempts}): {msg}"
177
217
  )
@@ -193,6 +233,143 @@ async def refresh_and_validate_youtube(
193
233
  return False
194
234
 
195
235
 
236
+ @dataclass
237
+ class YouTubeTimers:
238
+ """Event-loop timestamps for maintain_youtube. Defaults mean "never"."""
239
+
240
+ last_refresh: float = float("-inf")
241
+ last_probe: float = float("-inf")
242
+ last_failure_alert: float = float("-inf")
243
+ last_on_demand: float = float("-inf")
244
+
245
+
246
+ async def maintain_youtube(
247
+ browser: dict,
248
+ status: dict,
249
+ timers: YouTubeTimers,
250
+ now: float,
251
+ *,
252
+ probe=None,
253
+ refresh=None,
254
+ notify=None,
255
+ consume_request=None,
256
+ ) -> None:
257
+ """One keeper-loop pass for YouTube cookies.
258
+
259
+ Re-exports only when something says the cookies aren't working: a dead probe, a
260
+ bot block reported by the API, or (past the max age) an inconclusive probe. The
261
+ first pass after startup always exports, because restarting the keeper is the
262
+ documented way to force fresh cookies.
263
+ """
264
+ probe = probe or probe_cookies
265
+ refresh = refresh or refresh_and_validate_youtube
266
+ notify = notify or _send_notification
267
+ consume_request = consume_request or consume_refresh_request
268
+
269
+ refresh_needed = timers.last_refresh == float("-inf")
270
+ relaunch_first = False
271
+
272
+ # Always consume, so a request dropped by the cooldown can't fire later.
273
+ if consume_request():
274
+ if now - timers.last_on_demand >= YOUTUBE_ON_DEMAND_COOLDOWN:
275
+ logger.warning("Cookie refresh requested by the API after a YouTube bot block")
276
+ timers.last_on_demand = now
277
+ _record_youtube_event(status, "on_demand_refreshes")
278
+ relaunch_first = not refresh_needed
279
+ refresh_needed = True
280
+ else:
281
+ logger.info("Ignoring cookie refresh request: inside the on-demand cooldown")
282
+
283
+ if not refresh_needed and now - timers.last_probe >= YOUTUBE_PROBE_INTERVAL:
284
+ outcome, msg = await probe(get_youtube_cookies_path())
285
+ status.setdefault("youtube", {}).update({
286
+ "last_probe": datetime.now(timezone.utc).isoformat(),
287
+ "last_probe_status": outcome.value,
288
+ "last_probe_message": msg,
289
+ })
290
+ timers.last_probe = now
291
+ if outcome is ProbeOutcome.INVALID_COOKIES:
292
+ _record_youtube_event(status, "dead_probes")
293
+ logger.warning(f"Periodic probe found dead cookies — remediating: {msg}")
294
+ refresh_needed = True
295
+ relaunch_first = True
296
+ else:
297
+ if outcome is ProbeOutcome.OK:
298
+ _record_youtube_event(status, "ok_probes")
299
+ logger.info(f"Periodic cookie probe: {msg}")
300
+ refresh_needed = (
301
+ outcome is ProbeOutcome.TRANSIENT
302
+ and now - timers.last_refresh >= YOUTUBE_REFRESH_INTERVAL
303
+ )
304
+ _save_status(status)
305
+
306
+ if refresh_needed:
307
+ logger.info("--- YouTube cookie refresh ---")
308
+ # relaunch_first: this session's export is known dead, so re-exporting from
309
+ # it would be a no-op. Start from a fresh browser.
310
+ success = await refresh(browser, status, relaunch_first=relaunch_first)
311
+ status.setdefault("youtube", {}).update({
312
+ "last_refresh": datetime.now(timezone.utc).isoformat(),
313
+ "last_refresh_status": "ok" if success else "error",
314
+ })
315
+ if success:
316
+ if NOTIFY_ON_SUCCESS:
317
+ probe_ok = status.get("youtube", {}).get("last_probe_status") == "ok"
318
+ await notify(
319
+ "✅ YouTube Cookies Refreshed",
320
+ "YouTube cookies extracted, uploaded, and probe-validated."
321
+ if probe_ok
322
+ else "YouTube cookies extracted and uploaded (probe inconclusive).",
323
+ )
324
+ elif now - timers.last_failure_alert >= YOUTUBE_FAILURE_ALERT_INTERVAL:
325
+ # Throttled: a persistently-dead export re-remediates every
326
+ # probe cycle; without a cooldown that's ~48 pushes/day.
327
+ timers.last_failure_alert = now
328
+ await notify(
329
+ "❌ YouTube Cookies Invalid After Self-Heal",
330
+ "Cookie export still failing the yt-dlp probe after "
331
+ f"{YOUTUBE_MAX_REFRESH_ATTEMPTS} attempts (with browser "
332
+ "restarts). Manual intervention needed: "
333
+ f"{status.get('youtube', {}).get('last_failure_reason', 'unknown')}",
334
+ )
335
+
336
+ timers.last_refresh = now
337
+ timers.last_probe = now
338
+ _save_status(status)
339
+
340
+ await _alert_on_churn(status, notify)
341
+
342
+
343
+ async def _alert_on_churn(status: dict, notify) -> None:
344
+ """Notify once a UTC day when dead probes pass the threshold."""
345
+ stats = status.get("youtube", {}).get("daily_stats")
346
+ if not stats or stats["churn_alerted"]:
347
+ return
348
+ dead = stats["dead_probes"] + stats["rejected_exports"]
349
+ if dead <= YOUTUBE_CHURN_ALERT_THRESHOLD:
350
+ return
351
+ stats["churn_alerted"] = True
352
+ _save_status(status)
353
+ await notify(
354
+ "⚠️ YouTube Cookie Churn",
355
+ f"{dead} dead cookie probes so far today (limit {YOUTUBE_CHURN_ALERT_THRESHOLD}). "
356
+ "Self-heal is replacing them, but YouTube searches and downloads can fail until "
357
+ "it does, and searches return no YouTube results. Check "
358
+ "/var/log/flacfetch-credential-keeper.log on flacup and deploy/README.md "
359
+ "§ YouTube cookie health.",
360
+ )
361
+
362
+
363
+ async def _idle(seconds: float, poll: float = 2.0) -> None:
364
+ """Sleep between passes, waking early if the API has requested a refresh."""
365
+ loop = asyncio.get_running_loop()
366
+ deadline = loop.time() + seconds
367
+ while (remaining := deadline - loop.time()) > 0:
368
+ if refresh_requested():
369
+ return
370
+ await asyncio.sleep(min(poll, remaining))
371
+
372
+
196
373
  async def run_keeper():
197
374
  """Main keeper loop. Launches browser and periodically refreshes credentials."""
198
375
  _setup_logging()
@@ -228,80 +405,19 @@ async def run_keeper():
228
405
  _save_status(status)
229
406
 
230
407
  logger.info(
231
- f"Keeper running. YouTube every {YOUTUBE_REFRESH_INTERVAL // 3600}h, "
408
+ f"Keeper running. YouTube probe every {YOUTUBE_PROBE_INTERVAL // 60}m, "
232
409
  f"Spotify every {SPOTIFY_REFRESH_INTERVAL // 3600}h"
233
410
  )
234
411
 
235
412
  # Initialize to trigger immediately on first loop iteration
236
- last_youtube_refresh = float("-inf")
237
- last_youtube_probe = float("-inf")
238
- last_youtube_failure_alert = float("-inf")
413
+ youtube_timers = YouTubeTimers()
239
414
  last_spotify_refresh = float("-inf")
240
415
  last_librespot_refresh = float("-inf")
241
416
 
242
417
  while True:
243
418
  now = asyncio.get_event_loop().time()
244
419
 
245
- # Between refreshes, validate the exported cookies with a real
246
- # yt-dlp probe — a "logged in" browser is not proof the export
247
- # still works. A failed probe forces an immediate refresh cycle.
248
- refresh_due = now - last_youtube_refresh >= YOUTUBE_REFRESH_INTERVAL
249
- probe_found_dead = False
250
- if not refresh_due and now - last_youtube_probe >= YOUTUBE_PROBE_INTERVAL:
251
- outcome, msg = await probe_cookies(get_youtube_cookies_path())
252
- status.setdefault("youtube", {}).update({
253
- "last_probe": datetime.now(timezone.utc).isoformat(),
254
- "last_probe_status": outcome.value,
255
- "last_probe_message": msg,
256
- })
257
- last_youtube_probe = now
258
- _save_status(status)
259
- if outcome is ProbeOutcome.INVALID_COOKIES:
260
- logger.warning(f"Periodic probe found dead cookies — remediating: {msg}")
261
- refresh_due = True
262
- probe_found_dead = True
263
- else:
264
- logger.info(f"Periodic cookie probe: {msg}")
265
-
266
- # YouTube cookie refresh (+ probe validation with browser-restart
267
- # self-heal — see refresh_and_validate_youtube)
268
- if refresh_due:
269
- logger.info("--- YouTube cookie refresh ---")
270
-
271
- # If the periodic probe just proved the current session's
272
- # export dead, re-exporting from it is a guaranteed no-op —
273
- # start from a fresh browser.
274
- success = await refresh_and_validate_youtube(
275
- browser, status, relaunch_first=probe_found_dead
276
- )
277
- status.setdefault("youtube", {}).update({
278
- "last_refresh": datetime.now(timezone.utc).isoformat(),
279
- "last_refresh_status": "ok" if success else "error",
280
- })
281
- if success:
282
- if NOTIFY_ON_SUCCESS:
283
- probe_ok = status.get("youtube", {}).get("last_probe_status") == "ok"
284
- await _send_notification(
285
- "✅ YouTube Cookies Refreshed",
286
- "YouTube cookies extracted, uploaded, and probe-validated."
287
- if probe_ok
288
- else "YouTube cookies extracted and uploaded (probe inconclusive).",
289
- )
290
- elif now - last_youtube_failure_alert >= YOUTUBE_FAILURE_ALERT_INTERVAL:
291
- # Throttled: a persistently-dead export re-remediates every
292
- # probe cycle; without a cooldown that's ~48 pushes/day.
293
- last_youtube_failure_alert = now
294
- await _send_notification(
295
- "❌ YouTube Cookies Invalid After Self-Heal",
296
- "Cookie export still failing the yt-dlp probe after "
297
- f"{YOUTUBE_MAX_REFRESH_ATTEMPTS} attempts (with browser "
298
- "restarts). Manual intervention needed: "
299
- f"{status.get('youtube', {}).get('last_failure_reason', 'unknown')}",
300
- )
301
-
302
- last_youtube_refresh = now
303
- last_youtube_probe = now
304
- _save_status(status)
420
+ await maintain_youtube(browser, status, youtube_timers, now)
305
421
 
306
422
  # Spotify token refresh
307
423
  if now - last_spotify_refresh >= SPOTIFY_REFRESH_INTERVAL:
@@ -381,7 +497,7 @@ async def run_keeper():
381
497
  _save_status(status)
382
498
 
383
499
  # Sleep before next check (1 minute intervals)
384
- await asyncio.sleep(60)
500
+ await _idle(60)
385
501
 
386
502
  except Exception as e:
387
503
  logger.error(f"Credential keeper crashed: {e}", exc_info=True)
@@ -0,0 +1,113 @@
1
+ """Signal from the flacfetch API to the credential keeper: "these cookies just failed".
2
+
3
+ The API and the keeper are separate systemd services with the same user, so a file
4
+ next to the keeper's status file is enough. The API writes it when YouTube bot-blocks
5
+ a real search or download, the keeper notices within a couple of seconds and
6
+ re-exports, and the API watches the cookie file's mtime to know when to retry.
7
+
8
+ Standard library only: this is imported by the downloader and provider modules.
9
+ """
10
+ import logging
11
+ import os
12
+ import threading
13
+ import time
14
+ from typing import Optional
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ REQUEST_FILENAME = "youtube-refresh-request"
19
+ _DEFAULT_STATUS_FILE = "/mnt/flacfetch-data/browser-profiles/keeper-status.json"
20
+
21
+ # How long a bot-blocked search or download waits for new cookies. A relaunch and
22
+ # re-export takes about 12s on flacup, plus however long the keeper is busy.
23
+ WAIT_SECONDS = float(os.environ.get("FLACFETCH_COOKIE_REFRESH_WAIT", 30))
24
+ # At most one on-demand re-export per this many seconds, however many requests
25
+ # fail. Enforced on both sides: here so callers don't wait for a refresh that
26
+ # isn't coming, and in the keeper because it is the one doing the work.
27
+ COOLDOWN_SECONDS = float(os.environ.get("KEEPER_YOUTUBE_ON_DEMAND_COOLDOWN", 300))
28
+
29
+ _lock = threading.Lock()
30
+ _last_request_at: Optional[float] = None
31
+ _wait_until = 0.0
32
+
33
+
34
+ def _request_path() -> str:
35
+ status_file = os.environ.get("KEEPER_STATUS_FILE", _DEFAULT_STATUS_FILE)
36
+ return os.path.join(os.path.dirname(status_file), REQUEST_FILENAME)
37
+
38
+
39
+ def cookies_mtime(cookies_file: Optional[str]) -> int:
40
+ """The cookie file's mtime in ns, or 0 when there isn't one."""
41
+ if not cookies_file:
42
+ return 0
43
+ try:
44
+ return os.stat(cookies_file).st_mtime_ns
45
+ except OSError:
46
+ return 0
47
+
48
+
49
+ def request_refresh(wait_seconds: Optional[float] = None) -> bool:
50
+ """Ask the keeper for fresh cookies. Returns True if a new request was written.
51
+
52
+ Never raises. False means the request was rate limited or couldn't be written
53
+ (for example on a dev machine with no keeper).
54
+ """
55
+ global _last_request_at, _wait_until
56
+ with _lock:
57
+ now = time.monotonic()
58
+ if _last_request_at is not None and now - _last_request_at < COOLDOWN_SECONDS:
59
+ return False
60
+ try:
61
+ with open(_request_path(), "w") as f:
62
+ f.write(f"{time.time()}\n")
63
+ except OSError as e:
64
+ logger.info(f"Could not request a cookie refresh: {e}")
65
+ _wait_until = 0.0
66
+ return False
67
+ _last_request_at = now
68
+ _wait_until = now + (WAIT_SECONDS if wait_seconds is None else wait_seconds)
69
+ return True
70
+
71
+
72
+ def wait_for_fresh_cookies(
73
+ cookies_file: Optional[str],
74
+ baseline_mtime: int,
75
+ timeout: Optional[float] = None,
76
+ poll: float = 0.5,
77
+ ) -> bool:
78
+ """Block until the cookie file is newer than ``baseline_mtime``, requesting a
79
+ refresh if nobody has. Returns False if no new cookies arrive in time.
80
+
81
+ Take the baseline before the attempt that failed, so cookies replaced during
82
+ that attempt count. A caller that arrives while another caller's request is in
83
+ flight shares its deadline; one that arrives later in the cooldown returns at
84
+ once rather than waiting for a refresh that won't happen.
85
+
86
+ Blocks the calling thread: run it off the event loop, since the keeper delivers
87
+ new cookies through the API.
88
+ """
89
+ if not cookies_file:
90
+ return False
91
+ if cookies_mtime(cookies_file) > baseline_mtime:
92
+ return True
93
+
94
+ request_refresh(timeout)
95
+ while time.monotonic() < _wait_until:
96
+ if cookies_mtime(cookies_file) > baseline_mtime:
97
+ return True
98
+ time.sleep(poll)
99
+ return cookies_mtime(cookies_file) > baseline_mtime
100
+
101
+
102
+ def refresh_requested() -> bool:
103
+ """Whether a request is waiting. For the keeper's idle loop."""
104
+ return os.path.exists(_request_path())
105
+
106
+
107
+ def consume_refresh_request() -> bool:
108
+ """Remove a waiting request. Returns True if there was one. Never raises."""
109
+ try:
110
+ os.unlink(_request_path())
111
+ return True
112
+ except OSError:
113
+ return False
@@ -82,6 +82,11 @@ async def refresh_youtube_cookies(page, context) -> bool:
82
82
 
83
83
  Assumes the browser is already logged into Google.
84
84
  Returns True if cookies were successfully refreshed and uploaded.
85
+
86
+ Always leaves youtube.com afterwards. An export kept working when the tab left
87
+ YouTube within about 15 minutes and died within 30 when it stayed there (see
88
+ deploy/README.md), consistent with the open tab rotating the session and
89
+ invalidating the snapshot we just exported.
85
90
  """
86
91
  try:
87
92
  # Visit YouTube to refresh session cookies
@@ -102,3 +107,9 @@ async def refresh_youtube_cookies(page, context) -> bool:
102
107
  except Exception as e:
103
108
  logger.error(f"YouTube cookie refresh failed: {e}")
104
109
  return False
110
+
111
+ finally:
112
+ try:
113
+ await page.goto("about:blank", timeout=10000)
114
+ except Exception as e:
115
+ logger.warning(f"Could not leave YouTube after the cookie export: {e}")
@@ -37,6 +37,15 @@ SAMPLE_RATE = 44100
37
37
  CHANNELS = 2
38
38
  BIT_DEPTH = 16
39
39
 
40
+ # Used when a track's duration can't be resolved (see _resolve_duration_secs).
41
+ DEFAULT_DURATION_SECS = 300
42
+
43
+ # A capture longer than the track by more than this is refused: it means
44
+ # librespot played on past the requested track (e.g. Spotify autoplay queued a
45
+ # "similar" track), so the PCM contains someone else's audio. Durations are
46
+ # floored to whole seconds, so allow a small margin.
47
+ MAX_CAPTURE_OVERRUN_SECS = 5
48
+
40
49
  # Serialize all Spotify captures across the process.
41
50
  #
42
51
  # A Spotify download works by registering a librespot Spotify Connect device on
@@ -153,6 +162,11 @@ class SpotifyDownloader(Downloader):
153
162
  def _resolve_duration_secs(
154
163
  self, sp: "spotipy.Spotify", track_id: str, release: Release
155
164
  ) -> int:
165
+ return self._resolve_duration(sp, track_id, release)[0]
166
+
167
+ def _resolve_duration(
168
+ self, sp: "spotipy.Spotify", track_id: str, release: Release
169
+ ) -> tuple[int, bool]:
156
170
  """Resolve track duration in seconds.
157
171
 
158
172
  The PCM capture loop compares captured bytes against an expected size
@@ -164,18 +178,20 @@ class SpotifyDownloader(Downloader):
164
178
  Prefer the duration already on the release (from search). Otherwise
165
179
  look it up via the Web API, which is reliable and cheap. Only fall
166
180
  back to the 300s default if both fail.
181
+
182
+ Returns ``(seconds, known)``; ``known`` is False for the default guess.
167
183
  """
168
184
  if release.duration_seconds:
169
- return release.duration_seconds
185
+ return release.duration_seconds, True
170
186
  try:
171
187
  duration_ms = sp.track(track_id).get("duration_ms")
172
188
  if duration_ms:
173
189
  secs = duration_ms // 1000
174
190
  logger.debug(f"Resolved duration from Spotify API: {secs}s")
175
- return secs
191
+ return secs, True
176
192
  except Exception as e:
177
193
  logger.warning(f"Failed to look up track duration, using default: {e}")
178
- return 300
194
+ return DEFAULT_DURATION_SECS, False
179
195
 
180
196
  def download(
181
197
  self,
@@ -246,7 +262,7 @@ class SpotifyDownloader(Downloader):
246
262
  except Exception as e:
247
263
  raise SpotifyDownloadError(f"Authentication failed: {e}") from e
248
264
 
249
- duration_secs = self._resolve_duration_secs(sp, track_id, release)
265
+ duration_secs, duration_known = self._resolve_duration(sp, track_id, release)
250
266
 
251
267
  # Unique device name per capture. A single Spotify account can register
252
268
  # several Connect devices at once, so if a previous librespot has not
@@ -267,6 +283,12 @@ class SpotifyDownloader(Downloader):
267
283
  "--backend", "pipe",
268
284
  "--bitrate", "320",
269
285
  "--disable-discovery",
286
+ # Never play past the requested track. librespot decodes faster than
287
+ # real time, so anything autoplay queues after our track would be
288
+ # piped straight into this capture. Since Spotify started sending
289
+ # Web API plays as a "spotify:webapi:<id>" context (2026-11), a
290
+ # librespot that handles that context also autoplays after it.
291
+ "--autoplay", "off",
270
292
  # Volume settings for full-quality recording:
271
293
  "--initial-volume", "100", # Start at max volume
272
294
  "--volume-ctrl", "fixed", # No dynamic volume adjustments
@@ -383,6 +405,19 @@ class SpotifyDownloader(Downloader):
383
405
  captured_s = pcm_size / (SAMPLE_RATE * 4)
384
406
  logger.debug(f"Captured {pcm_size / (1024*1024):.2f} MB PCM")
385
407
 
408
+ # Refuse a capture that ran on past the requested track: the extra
409
+ # audio is a different song. Skipped when the duration is only the
410
+ # default guess.
411
+ if (
412
+ duration_known
413
+ and captured_s > duration_secs + MAX_CAPTURE_OVERRUN_SECS
414
+ ):
415
+ raise SpotifyDownloadError(
416
+ f"Captured {captured_s:.1f}s of audio but {track_uri} is "
417
+ f"{duration_secs}s long; librespot played past the track, "
418
+ "refusing to save audio that includes another song"
419
+ )
420
+
386
421
  # Convert PCM to a per-download temp FLAC, then atomically move it
387
422
  # into place. The atomic rename guarantees the returned path is
388
423
  # always a complete file even if a concurrent download targets the