flacfetch 0.34.0__tar.gz → 0.35.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flacfetch-0.34.0/flacfetch.egg-info → flacfetch-0.35.0}/PKG-INFO +1 -1
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/main.py +2 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/models.py +27 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/__init__.py +2 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/download.py +50 -1
- flacfetch-0.35.0/flacfetch/api/routes/spotify.py +48 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/download_manager.py +44 -1
- flacfetch-0.35.0/flacfetch/api/services/spotify_budget.py +274 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/album_match.py +6 -4
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/downloaders/spotify.py +28 -1
- {flacfetch-0.34.0 → flacfetch-0.35.0/flacfetch.egg-info}/PKG-INFO +1 -1
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch.egg-info/SOURCES.txt +3 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/pyproject.toml +1 -1
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_album_search.py +10 -0
- flacfetch-0.35.0/tests/test_api_routes.py +457 -0
- flacfetch-0.35.0/tests/test_spotify_budget.py +158 -0
- flacfetch-0.34.0/tests/test_api_routes.py +0 -192
- {flacfetch-0.34.0 → flacfetch-0.35.0}/ARCHITECTURE.md +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/LICENSE +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/MANIFEST.in +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/PLAN.md +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/README.md +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/__init__.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/__init__.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/auth.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/cache.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/config.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/credentials.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/health.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/search.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/routes/torrents.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/__init__.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/credential_check.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/disk_manager.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/health_check.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/search_cache.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/api/services/secret_manager_utils.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/categorize.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/config.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/interfaces.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/log.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/manager.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/matching.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/core/models.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/__init__.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/__main__.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/browser.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/google_login.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/keeper.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/librespot.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/probe.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/spotify.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/credential_keeper/youtube.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/downloaders/torrent.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/downloaders/torrent_coordinator.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/downloaders/youtube.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/interface/cli.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/interface/cli_remote.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/providers/gazelle.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/providers/ops.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/providers/red.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/providers/spotify.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch/providers/youtube.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch.egg-info/dependency_links.txt +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch.egg-info/entry_points.txt +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch.egg-info/requires.txt +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/flacfetch.egg-info/top_level.txt +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/requirements.txt +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/setup.cfg +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/setup.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/_route_paths.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_auth.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_config.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_deep_health.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_health_ytdlp.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_integration.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_models.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_api_services.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_cache_api.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_cli_cookies.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_cli_remote.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_core.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_credential_keeper.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_gazelle.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_keeper_probe.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_logging.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_manager.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_matching.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_models.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_ops.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_provider_errors.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_red.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_search_api.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_spotify.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_torrent_coordinator.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_torrent_downloader.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_torrent_shared_album.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_transcode_hf_noise.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_youtube.py +0 -0
- {flacfetch-0.34.0 → flacfetch-0.35.0}/tests/test_youtube_cookies.py +0 -0
|
@@ -33,6 +33,7 @@ from .routes import (
|
|
|
33
33
|
download_router,
|
|
34
34
|
health_router,
|
|
35
35
|
search_router,
|
|
36
|
+
spotify_router,
|
|
36
37
|
torrents_router,
|
|
37
38
|
)
|
|
38
39
|
from .services import get_disk_manager, get_download_manager, get_search_cache_service, set_server_started_at
|
|
@@ -167,6 +168,7 @@ def create_app() -> FastAPI:
|
|
|
167
168
|
app.include_router(config_router)
|
|
168
169
|
app.include_router(credentials_router)
|
|
169
170
|
app.include_router(cache_router)
|
|
171
|
+
app.include_router(spotify_router)
|
|
170
172
|
|
|
171
173
|
return app
|
|
172
174
|
|
|
@@ -7,6 +7,7 @@ from typing import (
|
|
|
7
7
|
Any,
|
|
8
8
|
Dict,
|
|
9
9
|
List,
|
|
10
|
+
Literal,
|
|
10
11
|
Optional,
|
|
11
12
|
)
|
|
12
13
|
|
|
@@ -158,6 +159,32 @@ class DownloadByIdRequest(BaseModel):
|
|
|
158
159
|
description="Torrents only: abort after this many seconds with no progress "
|
|
159
160
|
"(overrides FLACFETCH_MAX_STALL_SECONDS; capped at 3600).",
|
|
160
161
|
)
|
|
162
|
+
priority: Literal["interactive", "batch"] = Field(
|
|
163
|
+
"interactive",
|
|
164
|
+
description="Spotify only. 'batch' downloads are gated by the rolling listening budget "
|
|
165
|
+
"(429 with Retry-After when refused); 'interactive' are never refused.",
|
|
166
|
+
)
|
|
167
|
+
expected_duration_s: Optional[int] = Field(
|
|
168
|
+
None, gt=0, le=1200,
|
|
169
|
+
description="Spotify: expected track length in seconds. Required for priority='batch'.",
|
|
170
|
+
)
|
|
171
|
+
delete_local_after_upload: bool = Field(
|
|
172
|
+
False,
|
|
173
|
+
description="Delete the local file after a successful GCS upload (always on for batch).",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class SpotifyBudgetResponse(BaseModel):
|
|
178
|
+
"""Rolling-window Spotify listening usage."""
|
|
179
|
+
window_s: int
|
|
180
|
+
total_s: float
|
|
181
|
+
batch_s: float
|
|
182
|
+
batch_budget_s: int
|
|
183
|
+
hard_cap_s: int
|
|
184
|
+
batch_in_flight: int
|
|
185
|
+
interactive_in_flight: int
|
|
186
|
+
batch_allowed_now: bool
|
|
187
|
+
retry_after_s: int
|
|
161
188
|
|
|
162
189
|
|
|
163
190
|
class DownloadStatus(str, Enum):
|
|
@@ -7,6 +7,7 @@ from .credentials import router as credentials_router
|
|
|
7
7
|
from .download import router as download_router
|
|
8
8
|
from .health import router as health_router
|
|
9
9
|
from .search import router as search_router
|
|
10
|
+
from .spotify import router as spotify_router
|
|
10
11
|
from .torrents import router as torrents_router
|
|
11
12
|
|
|
12
13
|
__all__ = [
|
|
@@ -17,5 +18,6 @@ __all__ = [
|
|
|
17
18
|
"config_router",
|
|
18
19
|
"credentials_router",
|
|
19
20
|
"cache_router",
|
|
21
|
+
"spotify_router",
|
|
20
22
|
]
|
|
21
23
|
|
|
@@ -5,7 +5,7 @@ import logging
|
|
|
5
5
|
import os
|
|
6
6
|
|
|
7
7
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
|
8
|
-
from fastapi.responses import FileResponse
|
|
8
|
+
from fastapi.responses import FileResponse, JSONResponse
|
|
9
9
|
|
|
10
10
|
from ..auth import verify_api_key
|
|
11
11
|
from ..models import (
|
|
@@ -18,6 +18,7 @@ from ..models import (
|
|
|
18
18
|
DownloadStatusResponse,
|
|
19
19
|
)
|
|
20
20
|
from ..services import get_download_manager
|
|
21
|
+
from ..services.spotify_budget import DEFAULT_INTERACTIVE_RESERVE_S, BudgetConfig, get_spotify_ledger
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
23
24
|
router = APIRouter(tags=["download"])
|
|
@@ -110,6 +111,11 @@ async def start_download_by_id(
|
|
|
110
111
|
- download_url: For YouTube/Spotify, the direct URL (optional, constructed from source_id if missing).
|
|
111
112
|
REQUIRED for source_name='URL' (any yt-dlp-supported site, e.g. Facebook, SoundCloud, TikTok).
|
|
112
113
|
|
|
114
|
+
- priority / expected_duration_s (Spotify): 'batch' downloads need expected_duration_s and are
|
|
115
|
+
refused with 429 {"detail": {"code", "retry_after_s"}} + Retry-After when the rolling Spotify
|
|
116
|
+
listening budget is exhausted or the batch gate is closed. 'interactive' is never refused.
|
|
117
|
+
- delete_local_after_upload: delete the local file after a successful GCS upload (always for batch).
|
|
118
|
+
|
|
113
119
|
The download runs in the background. Use GET /download/{download_id}/status to check progress.
|
|
114
120
|
"""
|
|
115
121
|
manager = get_download_manager()
|
|
@@ -137,6 +143,14 @@ async def start_download_by_id(
|
|
|
137
143
|
detail="gcs_path is required when upload_to_gcs is true"
|
|
138
144
|
)
|
|
139
145
|
|
|
146
|
+
is_spotify = request.source_name == "Spotify"
|
|
147
|
+
is_batch = is_spotify and request.priority == "batch"
|
|
148
|
+
if is_batch and not request.expected_duration_s:
|
|
149
|
+
raise HTTPException(
|
|
150
|
+
status_code=400,
|
|
151
|
+
detail="expected_duration_s is required for Spotify downloads with priority='batch'"
|
|
152
|
+
)
|
|
153
|
+
|
|
140
154
|
logger.info(
|
|
141
155
|
f"Download by ID requested: source={request.source_name}, "
|
|
142
156
|
f"id={request.source_id}, target_file={request.target_file}"
|
|
@@ -152,8 +166,43 @@ async def start_download_by_id(
|
|
|
152
166
|
upload_to_gcs=request.upload_to_gcs,
|
|
153
167
|
gcs_destination=request.gcs_path,
|
|
154
168
|
max_stall_seconds=request.max_stall_seconds,
|
|
169
|
+
delete_local_after_upload=request.delete_local_after_upload or is_batch,
|
|
155
170
|
)
|
|
156
171
|
|
|
172
|
+
if is_spotify:
|
|
173
|
+
try:
|
|
174
|
+
ledger = get_spotify_ledger()
|
|
175
|
+
if is_batch:
|
|
176
|
+
ok, code, retry_after = ledger.try_reserve_batch(
|
|
177
|
+
task.download_id, request.expected_duration_s, BudgetConfig.from_env()
|
|
178
|
+
)
|
|
179
|
+
if not ok:
|
|
180
|
+
manager.discard_download(task.download_id)
|
|
181
|
+
logger.warning(f"Spotify batch download refused: {code} (retry in {retry_after}s)")
|
|
182
|
+
return JSONResponse(
|
|
183
|
+
status_code=429,
|
|
184
|
+
content={"detail": {"code": code, "retry_after_s": retry_after}},
|
|
185
|
+
headers={"Retry-After": str(retry_after)},
|
|
186
|
+
)
|
|
187
|
+
else:
|
|
188
|
+
ledger.reserve(
|
|
189
|
+
task.download_id, "interactive",
|
|
190
|
+
request.expected_duration_s or DEFAULT_INTERACTIVE_RESERVE_S,
|
|
191
|
+
)
|
|
192
|
+
task.spotify_reserved = True
|
|
193
|
+
except Exception as e:
|
|
194
|
+
if not is_batch:
|
|
195
|
+
# Interactive downloads are never refused: fail open, uncounted.
|
|
196
|
+
logger.warning(f"Spotify ledger unavailable; interactive download proceeds uncounted: {e}")
|
|
197
|
+
else:
|
|
198
|
+
manager.discard_download(task.download_id)
|
|
199
|
+
logger.warning(f"Spotify ledger unavailable; refusing batch download: {e}")
|
|
200
|
+
return JSONResponse(
|
|
201
|
+
status_code=503,
|
|
202
|
+
content={"detail": {"code": "budget_unavailable", "retry_after_s": 60}},
|
|
203
|
+
headers={"Retry-After": "60"},
|
|
204
|
+
)
|
|
205
|
+
|
|
157
206
|
logger.info(f"Created download-by-id task: {task.download_id} for {task.provider} ID={task.source_id}")
|
|
158
207
|
|
|
159
208
|
# Start download in background
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Spotify listening-budget endpoint.
|
|
3
|
+
"""
|
|
4
|
+
import logging
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from fastapi import APIRouter, Depends
|
|
8
|
+
from fastapi.responses import JSONResponse
|
|
9
|
+
|
|
10
|
+
from ..auth import verify_api_key
|
|
11
|
+
from ..models import SpotifyBudgetResponse
|
|
12
|
+
from ..services.spotify_budget import (
|
|
13
|
+
PROBE_BATCH_SECONDS,
|
|
14
|
+
BudgetConfig,
|
|
15
|
+
check_batch,
|
|
16
|
+
get_spotify_ledger,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
router = APIRouter(tags=["spotify"])
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@router.get("/spotify/budget", response_model=SpotifyBudgetResponse)
|
|
24
|
+
async def get_spotify_budget(api_key: str = Depends(verify_api_key)) -> SpotifyBudgetResponse:
|
|
25
|
+
"""Rolling-window Spotify listening usage and whether a batch download may start now."""
|
|
26
|
+
cfg = BudgetConfig.from_env()
|
|
27
|
+
now = time.time()
|
|
28
|
+
try:
|
|
29
|
+
usage = get_spotify_ledger().usage(now, cfg.window_s)
|
|
30
|
+
except Exception as e:
|
|
31
|
+
logger.warning(f"Spotify ledger unavailable for /spotify/budget: {e}")
|
|
32
|
+
return JSONResponse(
|
|
33
|
+
status_code=503,
|
|
34
|
+
content={"detail": {"code": "budget_unavailable", "retry_after_s": 60}},
|
|
35
|
+
headers={"Retry-After": "60"},
|
|
36
|
+
)
|
|
37
|
+
ok, _code, retry = check_batch(usage, PROBE_BATCH_SECONDS, cfg, now)
|
|
38
|
+
return SpotifyBudgetResponse(
|
|
39
|
+
window_s=cfg.window_s,
|
|
40
|
+
total_s=usage.total_s,
|
|
41
|
+
batch_s=usage.batch_s,
|
|
42
|
+
batch_budget_s=cfg.batch_budget_s,
|
|
43
|
+
hard_cap_s=cfg.hard_cap_s,
|
|
44
|
+
batch_in_flight=usage.batch_in_flight,
|
|
45
|
+
interactive_in_flight=usage.interactive_in_flight,
|
|
46
|
+
batch_allowed_now=ok,
|
|
47
|
+
retry_after_s=retry,
|
|
48
|
+
)
|
|
@@ -15,6 +15,7 @@ from typing import Any, Dict, List, Optional
|
|
|
15
15
|
|
|
16
16
|
from ...downloaders.torrent import TorrentStalledError
|
|
17
17
|
from ..models import DownloadStatus
|
|
18
|
+
from .spotify_budget import get_spotify_ledger
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(__name__)
|
|
20
21
|
|
|
@@ -49,6 +50,8 @@ class DownloadTask:
|
|
|
49
50
|
target_file: Optional[str] = None
|
|
50
51
|
download_url: Optional[str] = None
|
|
51
52
|
max_stall_seconds: Optional[float] = None # torrents: per-request stall ceiling
|
|
53
|
+
delete_local_after_upload: bool = False
|
|
54
|
+
spotify_reserved: bool = False # a Spotify budget ledger entry exists for this task
|
|
52
55
|
|
|
53
56
|
|
|
54
57
|
@dataclass
|
|
@@ -477,6 +480,7 @@ class DownloadManager:
|
|
|
477
480
|
upload_to_gcs: bool = False,
|
|
478
481
|
gcs_destination: Optional[str] = None,
|
|
479
482
|
max_stall_seconds: Optional[float] = None,
|
|
483
|
+
delete_local_after_upload: bool = False,
|
|
480
484
|
) -> DownloadTask:
|
|
481
485
|
"""
|
|
482
486
|
Create a download task for direct download by source ID (no search required).
|
|
@@ -496,6 +500,7 @@ class DownloadManager:
|
|
|
496
500
|
upload_to_gcs=upload_to_gcs,
|
|
497
501
|
gcs_destination=gcs_destination,
|
|
498
502
|
max_stall_seconds=max_stall_seconds,
|
|
503
|
+
delete_local_after_upload=delete_local_after_upload,
|
|
499
504
|
started_at=datetime.now(timezone.utc),
|
|
500
505
|
)
|
|
501
506
|
|
|
@@ -504,6 +509,35 @@ class DownloadManager:
|
|
|
504
509
|
|
|
505
510
|
return task
|
|
506
511
|
|
|
512
|
+
def discard_download(self, download_id: str) -> None:
|
|
513
|
+
"""Forget a task that was created but never started (e.g. refused by the budget)."""
|
|
514
|
+
with self._lock:
|
|
515
|
+
self._downloads.pop(download_id, None)
|
|
516
|
+
|
|
517
|
+
def _settle_spotify(self, task: DownloadTask, captured_s: Optional[float]) -> None:
|
|
518
|
+
"""Close out the task's Spotify ledger entry (settle with captured audio, else release)."""
|
|
519
|
+
if not task.spotify_reserved:
|
|
520
|
+
return
|
|
521
|
+
task.spotify_reserved = False
|
|
522
|
+
try:
|
|
523
|
+
ledger = get_spotify_ledger()
|
|
524
|
+
if captured_s is None or captured_s > 0:
|
|
525
|
+
ledger.settle(task.download_id, captured_s)
|
|
526
|
+
else:
|
|
527
|
+
ledger.release(task.download_id)
|
|
528
|
+
except Exception as e:
|
|
529
|
+
logger.warning(f"Could not update Spotify ledger for {task.download_id}: {e}")
|
|
530
|
+
|
|
531
|
+
def _delete_local_file(self, path: str) -> None:
|
|
532
|
+
"""Remove an uploaded file and its parent dir if that is now empty (never download_dir)."""
|
|
533
|
+
try:
|
|
534
|
+
os.unlink(path)
|
|
535
|
+
parent = os.path.dirname(path)
|
|
536
|
+
if os.path.abspath(parent) != os.path.abspath(self.download_dir) and not os.listdir(parent):
|
|
537
|
+
os.rmdir(parent)
|
|
538
|
+
except OSError as e:
|
|
539
|
+
logger.warning(f"Could not delete local file after upload {path}: {e}")
|
|
540
|
+
|
|
507
541
|
async def execute_download_by_id(self, download_id: str) -> None:
|
|
508
542
|
"""
|
|
509
543
|
Execute a download-by-id task (runs in background).
|
|
@@ -527,12 +561,17 @@ class DownloadManager:
|
|
|
527
561
|
# Execute download in a thread to avoid blocking the event loop
|
|
528
562
|
manager = self._get_fetch_manager()
|
|
529
563
|
loop = asyncio.get_running_loop()
|
|
564
|
+
# Never delete seeding torrent data; the flag only applies to non-torrent sources.
|
|
565
|
+
delete_after_upload = task.delete_local_after_upload and task.provider not in ("RED", "OPS")
|
|
566
|
+
# Concurrent downloads with the same output_filename must not share a file:
|
|
567
|
+
# use a per-task directory when the file is deleted after upload.
|
|
568
|
+
out_dir = os.path.join(self.download_dir, task.download_id) if delete_after_upload else self.download_dir
|
|
530
569
|
output_path = await loop.run_in_executor(
|
|
531
570
|
None,
|
|
532
571
|
lambda: manager.download_by_id(
|
|
533
572
|
source_name=task.provider,
|
|
534
573
|
source_id=task.source_id,
|
|
535
|
-
output_path=
|
|
574
|
+
output_path=out_dir,
|
|
536
575
|
output_filename=task.output_filename,
|
|
537
576
|
target_file=task.target_file,
|
|
538
577
|
download_url=task.download_url,
|
|
@@ -547,6 +586,7 @@ class DownloadManager:
|
|
|
547
586
|
)
|
|
548
587
|
|
|
549
588
|
logger.info(f"Download complete: {output_path}")
|
|
589
|
+
self._settle_spotify(task, getattr(output_path, "captured_s", None))
|
|
550
590
|
|
|
551
591
|
# Upload to GCS if requested
|
|
552
592
|
if task.upload_to_gcs and task.gcs_destination:
|
|
@@ -554,6 +594,8 @@ class DownloadManager:
|
|
|
554
594
|
gcs_path = await self._upload_to_gcs(output_path, task.gcs_destination)
|
|
555
595
|
self.update_download(download_id, gcs_path=gcs_path)
|
|
556
596
|
logger.info(f"Uploaded to GCS: {gcs_path}")
|
|
597
|
+
if delete_after_upload:
|
|
598
|
+
self._delete_local_file(output_path)
|
|
557
599
|
|
|
558
600
|
# Final status depends on whether it's a torrent (seeding) or not
|
|
559
601
|
if task.provider in ["RED", "OPS"] and self.keep_seeding:
|
|
@@ -565,6 +607,7 @@ class DownloadManager:
|
|
|
565
607
|
|
|
566
608
|
except Exception as e:
|
|
567
609
|
logger.error(f"Download by ID failed: {e}", exc_info=True)
|
|
610
|
+
self._settle_spotify(task, getattr(e, "captured_s", 0.0))
|
|
568
611
|
self.update_download(
|
|
569
612
|
download_id,
|
|
570
613
|
status=DownloadStatus.FAILED,
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Spotify listening budget for the flacfetch HTTP API.
|
|
3
|
+
|
|
4
|
+
Spotify Premium accounts must not "listen" to more than 24 h of audio per
|
|
5
|
+
rolling 24 h. Every Spotify download made through the API is recorded in a
|
|
6
|
+
small SQLite ledger (survives restarts). Batch (background, bulk) downloads
|
|
7
|
+
are gated against the ledger; interactive downloads are never refused, only
|
|
8
|
+
counted.
|
|
9
|
+
"""
|
|
10
|
+
import logging
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
import sqlite3
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Callable, Iterator, Optional, Tuple
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
# Reserved entries older than this are assumed to have crashed mid-flight and
|
|
23
|
+
# are counted at their reserved length.
|
|
24
|
+
STALE_RESERVED_S = 2 * 3600
|
|
25
|
+
|
|
26
|
+
# Capture format used by SpotifyDownloader: 44.1 kHz, 16-bit, stereo PCM.
|
|
27
|
+
PCM_BYTES_PER_SECOND = 44100 * 4
|
|
28
|
+
|
|
29
|
+
# Reservation used for an interactive download that gave no expected duration.
|
|
30
|
+
DEFAULT_INTERACTIVE_RESERVE_S = 300
|
|
31
|
+
|
|
32
|
+
# Typical track length used by GET /spotify/budget to answer "can a batch start?".
|
|
33
|
+
PROBE_BATCH_SECONDS = 240
|
|
34
|
+
|
|
35
|
+
CODE_BUDGET = "spotify_budget"
|
|
36
|
+
CODE_BATCH_BUSY = "batch_busy"
|
|
37
|
+
CODE_INTERACTIVE_ACTIVE = "interactive_active"
|
|
38
|
+
CODE_BATCH_GAP = "batch_gap"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _env_int(name: str, default: int) -> int:
|
|
42
|
+
raw = os.environ.get(name)
|
|
43
|
+
if not raw:
|
|
44
|
+
return default
|
|
45
|
+
try:
|
|
46
|
+
return int(raw)
|
|
47
|
+
except ValueError:
|
|
48
|
+
logger.warning(f"Ignoring non-integer {name}={raw!r}; using {default}")
|
|
49
|
+
return default
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class BudgetConfig:
|
|
54
|
+
batch_budget_s: int = 43200
|
|
55
|
+
hard_cap_s: int = 86400
|
|
56
|
+
interactive_reserve_s: int = 7200
|
|
57
|
+
max_batch_in_flight: int = 2
|
|
58
|
+
min_batch_gap_s: int = 30
|
|
59
|
+
window_s: int = 86400
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_env(cls) -> "BudgetConfig":
|
|
63
|
+
d = cls()
|
|
64
|
+
return cls(
|
|
65
|
+
batch_budget_s=_env_int("FLACFETCH_SPOTIFY_BATCH_BUDGET_S", d.batch_budget_s),
|
|
66
|
+
hard_cap_s=_env_int("FLACFETCH_SPOTIFY_HARD_CAP_S", d.hard_cap_s),
|
|
67
|
+
interactive_reserve_s=_env_int("FLACFETCH_SPOTIFY_INTERACTIVE_RESERVE_S", d.interactive_reserve_s),
|
|
68
|
+
max_batch_in_flight=_env_int("FLACFETCH_SPOTIFY_MAX_BATCH_IN_FLIGHT", d.max_batch_in_flight),
|
|
69
|
+
min_batch_gap_s=_env_int("FLACFETCH_SPOTIFY_MIN_BATCH_GAP_S", d.min_batch_gap_s),
|
|
70
|
+
window_s=_env_int("FLACFETCH_SPOTIFY_WINDOW_S", d.window_s),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class Usage:
|
|
76
|
+
total_s: float = 0.0
|
|
77
|
+
batch_s: float = 0.0
|
|
78
|
+
batch_in_flight: int = 0
|
|
79
|
+
interactive_in_flight: int = 0
|
|
80
|
+
last_batch_start: Optional[float] = None
|
|
81
|
+
# (started_at, counted_seconds, priority) of every counted entry; used to
|
|
82
|
+
# work out when enough usage ages out of the window.
|
|
83
|
+
entries: Tuple[Tuple[float, float, str], ...] = ()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _age_out_delay(
|
|
87
|
+
entries: Tuple[Tuple[float, float, str], ...],
|
|
88
|
+
excess: float,
|
|
89
|
+
only_batch: bool,
|
|
90
|
+
cfg: BudgetConfig,
|
|
91
|
+
now: float,
|
|
92
|
+
) -> float:
|
|
93
|
+
"""Seconds until entries aging out of the window free up `excess` seconds."""
|
|
94
|
+
freed = 0.0
|
|
95
|
+
for started_at, seconds, priority in sorted(entries):
|
|
96
|
+
if only_batch and priority != "batch":
|
|
97
|
+
continue
|
|
98
|
+
freed += seconds
|
|
99
|
+
if freed >= excess:
|
|
100
|
+
return max(0.0, started_at + cfg.window_s - now)
|
|
101
|
+
return float(cfg.window_s)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def check_batch(
|
|
105
|
+
usage: Usage, seconds: float, cfg: BudgetConfig, now: float
|
|
106
|
+
) -> Tuple[bool, Optional[str], int]:
|
|
107
|
+
"""Decide whether a batch download of `seconds` may start.
|
|
108
|
+
|
|
109
|
+
Returns (ok, code, retry_after_s).
|
|
110
|
+
"""
|
|
111
|
+
batch_excess = usage.batch_s + seconds - cfg.batch_budget_s
|
|
112
|
+
soft_cap = cfg.hard_cap_s - cfg.interactive_reserve_s
|
|
113
|
+
total_excess = usage.total_s + seconds - soft_cap
|
|
114
|
+
if batch_excess > 0 or total_excess > 0:
|
|
115
|
+
delay = 0.0
|
|
116
|
+
if batch_excess > 0:
|
|
117
|
+
delay = max(delay, _age_out_delay(usage.entries, batch_excess, True, cfg, now))
|
|
118
|
+
if total_excess > 0:
|
|
119
|
+
delay = max(delay, _age_out_delay(usage.entries, total_excess, False, cfg, now))
|
|
120
|
+
return False, CODE_BUDGET, max(1, math.ceil(delay))
|
|
121
|
+
|
|
122
|
+
if usage.batch_in_flight >= cfg.max_batch_in_flight:
|
|
123
|
+
return False, CODE_BATCH_BUSY, cfg.min_batch_gap_s
|
|
124
|
+
|
|
125
|
+
if usage.interactive_in_flight > 0:
|
|
126
|
+
return False, CODE_INTERACTIVE_ACTIVE, 60
|
|
127
|
+
|
|
128
|
+
if usage.last_batch_start is not None:
|
|
129
|
+
since = now - usage.last_batch_start
|
|
130
|
+
if since < cfg.min_batch_gap_s:
|
|
131
|
+
return False, CODE_BATCH_GAP, max(1, math.ceil(cfg.min_batch_gap_s - since))
|
|
132
|
+
|
|
133
|
+
return True, None, 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class SpotifyLedger:
|
|
137
|
+
"""SQLite ledger of Spotify listening time (one connection per call)."""
|
|
138
|
+
|
|
139
|
+
def __init__(self, path: str, now: Callable[[], float] = time.time):
|
|
140
|
+
self.path = path
|
|
141
|
+
self._now = now
|
|
142
|
+
directory = os.path.dirname(os.path.abspath(path))
|
|
143
|
+
os.makedirs(directory, exist_ok=True)
|
|
144
|
+
with self._tx() as conn:
|
|
145
|
+
conn.execute(
|
|
146
|
+
"CREATE TABLE IF NOT EXISTS entries ("
|
|
147
|
+
"download_id TEXT PRIMARY KEY, priority TEXT NOT NULL, "
|
|
148
|
+
"started_at REAL NOT NULL, reserved_s REAL NOT NULL, "
|
|
149
|
+
"captured_s REAL NULL, state TEXT NOT NULL)"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
@contextmanager
|
|
153
|
+
def _tx(self) -> Iterator[sqlite3.Connection]:
|
|
154
|
+
conn = sqlite3.connect(self.path, timeout=30, isolation_level=None)
|
|
155
|
+
try:
|
|
156
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
157
|
+
try:
|
|
158
|
+
yield conn
|
|
159
|
+
except BaseException:
|
|
160
|
+
conn.execute("ROLLBACK")
|
|
161
|
+
raise
|
|
162
|
+
else:
|
|
163
|
+
conn.execute("COMMIT")
|
|
164
|
+
finally:
|
|
165
|
+
conn.close()
|
|
166
|
+
|
|
167
|
+
def _insert(self, conn: sqlite3.Connection, download_id: str, priority: str, seconds: float) -> None:
|
|
168
|
+
conn.execute(
|
|
169
|
+
"INSERT OR REPLACE INTO entries(download_id, priority, started_at, reserved_s, captured_s, state) "
|
|
170
|
+
"VALUES (?, ?, ?, ?, NULL, 'reserved')",
|
|
171
|
+
(download_id, priority, self._now(), float(seconds)),
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
def reserve(self, download_id: str, priority: str, seconds: float) -> None:
|
|
175
|
+
with self._tx() as conn:
|
|
176
|
+
self._insert(conn, download_id, priority, seconds)
|
|
177
|
+
|
|
178
|
+
def try_reserve_batch(
|
|
179
|
+
self, download_id: str, seconds: float, cfg: BudgetConfig
|
|
180
|
+
) -> Tuple[bool, Optional[str], int]:
|
|
181
|
+
"""Atomically check the batch gate and, if open, reserve."""
|
|
182
|
+
with self._tx() as conn:
|
|
183
|
+
now = self._now()
|
|
184
|
+
usage = self._usage(conn, now, cfg.window_s)
|
|
185
|
+
ok, code, retry = check_batch(usage, seconds, cfg, now)
|
|
186
|
+
if ok:
|
|
187
|
+
self._insert(conn, download_id, "batch", seconds)
|
|
188
|
+
return ok, code, retry
|
|
189
|
+
|
|
190
|
+
def settle(self, download_id: str, captured_s: Optional[float]) -> None:
|
|
191
|
+
"""Record the captured length (None keeps the reservation as the count)."""
|
|
192
|
+
with self._tx() as conn:
|
|
193
|
+
conn.execute(
|
|
194
|
+
"UPDATE entries SET state='done', captured_s=COALESCE(?, reserved_s) WHERE download_id=?",
|
|
195
|
+
(None if captured_s is None else float(captured_s), download_id),
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
def release(self, download_id: str) -> None:
|
|
199
|
+
with self._tx() as conn:
|
|
200
|
+
conn.execute("UPDATE entries SET state='released' WHERE download_id=?", (download_id,))
|
|
201
|
+
|
|
202
|
+
def settle_orphans(self) -> int:
|
|
203
|
+
"""Settle every still-reserved row at its reserved size (their process is gone)."""
|
|
204
|
+
with self._tx() as conn:
|
|
205
|
+
cur = conn.execute(
|
|
206
|
+
"UPDATE entries SET state='done', captured_s=reserved_s WHERE state='reserved'"
|
|
207
|
+
)
|
|
208
|
+
return cur.rowcount
|
|
209
|
+
|
|
210
|
+
def usage(self, now: Optional[float] = None, window_s: int = 86400) -> Usage:
|
|
211
|
+
with self._tx() as conn:
|
|
212
|
+
return self._usage(conn, self._now() if now is None else now, window_s)
|
|
213
|
+
|
|
214
|
+
def _usage(self, conn: sqlite3.Connection, now: float, window_s: int) -> Usage:
|
|
215
|
+
# Crash safety: a reservation that never settled counts at its reserved size.
|
|
216
|
+
conn.execute(
|
|
217
|
+
"UPDATE entries SET state='done', captured_s=reserved_s "
|
|
218
|
+
"WHERE state='reserved' AND started_at < ?",
|
|
219
|
+
(now - STALE_RESERVED_S,),
|
|
220
|
+
)
|
|
221
|
+
rows = conn.execute(
|
|
222
|
+
"SELECT priority, started_at, reserved_s, captured_s, state FROM entries "
|
|
223
|
+
"WHERE state != 'released' AND started_at > ?",
|
|
224
|
+
(now - window_s,),
|
|
225
|
+
).fetchall()
|
|
226
|
+
usage = Usage()
|
|
227
|
+
entries = []
|
|
228
|
+
for priority, started_at, reserved_s, captured_s, state in rows:
|
|
229
|
+
counted = captured_s if (state == "done" and captured_s is not None) else reserved_s
|
|
230
|
+
usage.total_s += counted
|
|
231
|
+
entries.append((started_at, counted, priority))
|
|
232
|
+
if priority == "batch":
|
|
233
|
+
usage.batch_s += counted
|
|
234
|
+
if state == "reserved":
|
|
235
|
+
if priority == "batch":
|
|
236
|
+
usage.batch_in_flight += 1
|
|
237
|
+
else:
|
|
238
|
+
usage.interactive_in_flight += 1
|
|
239
|
+
last = conn.execute(
|
|
240
|
+
"SELECT MAX(started_at) FROM entries WHERE priority='batch' AND state != 'released'"
|
|
241
|
+
).fetchone()[0]
|
|
242
|
+
usage.last_batch_start = last
|
|
243
|
+
usage.entries = tuple(entries)
|
|
244
|
+
return usage
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
_ledger: Optional[SpotifyLedger] = None
|
|
248
|
+
_ledger_lock = threading.Lock()
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def default_ledger_path() -> str:
|
|
252
|
+
return os.environ.get("FLACFETCH_SPOTIFY_LEDGER") or os.path.join(
|
|
253
|
+
os.path.expanduser("~"), ".cache", "flacfetch", "spotify_ledger.sqlite"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def get_spotify_ledger() -> SpotifyLedger:
|
|
258
|
+
global _ledger
|
|
259
|
+
with _ledger_lock:
|
|
260
|
+
if _ledger is None:
|
|
261
|
+
ledger = SpotifyLedger(default_ledger_path())
|
|
262
|
+
# A new process has no downloads in flight; rows left reserved are orphans of a restart.
|
|
263
|
+
n = ledger.settle_orphans()
|
|
264
|
+
if n:
|
|
265
|
+
logger.info(f"Settled {n} orphaned reserved Spotify ledger entries at startup")
|
|
266
|
+
_ledger = ledger
|
|
267
|
+
return _ledger
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def reset_spotify_ledger() -> None:
|
|
271
|
+
"""Drop the cached singleton (tests / config changes)."""
|
|
272
|
+
global _ledger
|
|
273
|
+
with _ledger_lock:
|
|
274
|
+
_ledger = None
|
|
@@ -36,11 +36,11 @@ POSITION_BONUS = 0.05
|
|
|
36
36
|
|
|
37
37
|
# "06 Title", "06 - Title", "06. Title", "1-06 Title" (disc-track), "A1 Title" is ignored.
|
|
38
38
|
_TRACK_NUMBER_RE = re.compile(r"^(?:\d{1,2}[-.])?(\d{1,3})(?=[\s.\-_])")
|
|
39
|
-
# Bracketed suffixes that don't change which recording it is ("(feat. X)",
|
|
39
|
+
# Bracketed suffixes that don't change which recording it is ("(feat. X)", "(vs. X)",
|
|
40
40
|
# "[Remastered 2011]", "(Mono)"); stripped from titles and file names alike.
|
|
41
41
|
# Version suffixes ("(Live)", "(Remix)") are NOT stripped: they're another recording.
|
|
42
42
|
_NEUTRAL_BRACKET_RE = re.compile(
|
|
43
|
-
r"\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring|with|remaster(?:ed)?|\d{4}\s+remaster(?:ed)?|"
|
|
43
|
+
r"\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring|with|vs\.?|versus|remaster(?:ed)?|\d{4}\s+remaster(?:ed)?|"
|
|
44
44
|
r"mono|stereo|explicit|clean|bonus track)\b[^\)\]]*[\)\]]",
|
|
45
45
|
re.IGNORECASE,
|
|
46
46
|
)
|
|
@@ -130,10 +130,12 @@ def title_score(title: str, path: str) -> float:
|
|
|
130
130
|
"""How well a file matches a track title (0..1), ignoring neutral suffixes such
|
|
131
131
|
as "(feat. X)" on either side. 0.0 when the two disagree on version markers:
|
|
132
132
|
"Song (Live)" never matches "01 Song.flac", nor "Song" "02 Song (Live).flac"."""
|
|
133
|
+
# Markers are compared on the full names: a credit bracket can also name a
|
|
134
|
+
# version ("(vs. X Remix)") and stripping it must not hide that.
|
|
135
|
+
if version_markers(title) != version_markers(_file_title(path)):
|
|
136
|
+
return 0.0
|
|
133
137
|
want = _strip_neutral(title) or title
|
|
134
138
|
have = _strip_neutral(_file_title(path)) or _file_title(path)
|
|
135
|
-
if version_markers(want) != version_markers(have):
|
|
136
|
-
return 0.0
|
|
137
139
|
# calculate_match_score strips a file extension (everything after the last
|
|
138
140
|
# dot), so give it one: "05 Mr. Brightside" must not become "05 Mr".
|
|
139
141
|
return calculate_match_score(want, f"{have}.flac")
|