fmscrape 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fmscrape might be problematic. Click here for more details.
- fmscrape/__init__.py +80 -0
- fmscrape/api/__init__.py +2 -0
- fmscrape/api/album.py +80 -0
- fmscrape/api/artist.py +265 -0
- fmscrape/api/auth.py +21 -0
- fmscrape/api/base.py +185 -0
- fmscrape/api/chart.py +71 -0
- fmscrape/api/geo.py +22 -0
- fmscrape/api/library.py +272 -0
- fmscrape/api/tag.py +174 -0
- fmscrape/api/track.py +144 -0
- fmscrape/api/user.py +638 -0
- fmscrape/client.py +474 -0
- fmscrape/errors.py +37 -0
- fmscrape/models.py +594 -0
- fmscrape/normalize.py +80 -0
- fmscrape/parsers/__init__.py +2 -0
- fmscrape/parsers/album.py +152 -0
- fmscrape/parsers/artist.py +305 -0
- fmscrape/parsers/chart.py +118 -0
- fmscrape/parsers/common.py +120 -0
- fmscrape/parsers/tag.py +207 -0
- fmscrape/parsers/track.py +184 -0
- fmscrape/parsers/user_lists.py +462 -0
- fmscrape/parsers/user_recent.py +190 -0
- fmscrape-0.1.0.dist-info/METADATA +126 -0
- fmscrape-0.1.0.dist-info/RECORD +29 -0
- fmscrape-0.1.0.dist-info/WHEEL +5 -0
- fmscrape-0.1.0.dist-info/top_level.txt +1 -0
fmscrape/client.py
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
"""HTTP client with rate limiting, robots.txt compliance, TTL cache, and exponential backoff."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
from urllib.parse import urlencode, urljoin, urlparse
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
|
|
15
|
+
from fmscrape.errors import (
|
|
16
|
+
NetworkError,
|
|
17
|
+
NotFoundError,
|
|
18
|
+
NotSupportedError,
|
|
19
|
+
RateLimitError,
|
|
20
|
+
RobotsDisallowedError,
|
|
21
|
+
)
|
|
22
|
+
from fmscrape.normalize import BASE_URL
|
|
23
|
+
|
|
24
|
+
DEFAULT_USER_AGENT = (
|
|
25
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
26
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
27
|
+
"Chrome/131.0.0.0 Safari/537.36"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_RETRYABLE_STATUSES = {429, 500, 502, 503, 504, 600}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _build_url(path: str, params: dict[str, Any] | None = None) -> str:
|
|
34
|
+
base = path if path.startswith(("http://", "https://")) else urljoin(BASE_URL, path)
|
|
35
|
+
if not params:
|
|
36
|
+
return base
|
|
37
|
+
clean_params = {k: v for k, v in params.items() if v is not None}
|
|
38
|
+
if not clean_params:
|
|
39
|
+
return base
|
|
40
|
+
sep = "&" if "?" in base else "?"
|
|
41
|
+
return f"{base}{sep}{urlencode(clean_params)}"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
_USEFUL_COOKIES = frozenset({"sessionid", "csrftoken", "fsuid", "lfmanon"})
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _filter_useful_cookies(cookies: dict[str, str]) -> dict[str, str]:
|
|
48
|
+
useful = {k: v for k, v in cookies.items() if k in _USEFUL_COOKIES}
|
|
49
|
+
return useful if useful else cookies
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _parse_cookie_string(raw: str) -> dict[str, str]:
|
|
53
|
+
cookies: dict[str, str] = {}
|
|
54
|
+
for part in raw.replace("\n", ";").split(";"):
|
|
55
|
+
if "=" not in part:
|
|
56
|
+
continue
|
|
57
|
+
k, v = part.split("=", 1)
|
|
58
|
+
key = k.strip()
|
|
59
|
+
if key:
|
|
60
|
+
cookies[key] = v.strip().strip("'\"")
|
|
61
|
+
return _filter_useful_cookies(cookies)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _parse_cookie_file(path: Path) -> dict[str, str]:
|
|
65
|
+
text = path.read_text(encoding="utf-8").strip()
|
|
66
|
+
if not text:
|
|
67
|
+
return {}
|
|
68
|
+
if text.startswith(("{", "[")):
|
|
69
|
+
data = json.loads(text)
|
|
70
|
+
if isinstance(data, dict):
|
|
71
|
+
return _filter_useful_cookies({str(k): str(v) for k, v in data.items()})
|
|
72
|
+
if isinstance(data, list):
|
|
73
|
+
cookies = {
|
|
74
|
+
str(item["name"]): str(item["value"])
|
|
75
|
+
for item in data
|
|
76
|
+
if isinstance(item, dict) and "name" in item and "value" in item
|
|
77
|
+
}
|
|
78
|
+
return _filter_useful_cookies(cookies)
|
|
79
|
+
|
|
80
|
+
# Netscape cookies.txt (7 tab-separated columns) or key=value lines
|
|
81
|
+
netscape: dict[str, str] = {}
|
|
82
|
+
for line in text.splitlines():
|
|
83
|
+
line = line.strip()
|
|
84
|
+
if not line or line.startswith("#"):
|
|
85
|
+
continue
|
|
86
|
+
cols = line.split("\t")
|
|
87
|
+
if len(cols) >= 7:
|
|
88
|
+
netscape[cols[5].strip()] = cols[6].strip()
|
|
89
|
+
if netscape:
|
|
90
|
+
return _filter_useful_cookies(netscape)
|
|
91
|
+
return _parse_cookie_string(text)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def load_cookies(cookies: dict[str, str] | str | Path | None) -> dict[str, str] | None:
|
|
95
|
+
"""Resolve cookies from dict, raw header string, file path, or cwd cookies.txt/cookies.json."""
|
|
96
|
+
if cookies is None:
|
|
97
|
+
for candidate in ("cookies.txt", "cookies.json"):
|
|
98
|
+
p = Path.cwd() / candidate
|
|
99
|
+
if p.is_file():
|
|
100
|
+
parsed = _parse_cookie_file(p)
|
|
101
|
+
return parsed or None
|
|
102
|
+
return None
|
|
103
|
+
if isinstance(cookies, Path):
|
|
104
|
+
return _parse_cookie_file(cookies) or None
|
|
105
|
+
if isinstance(cookies, str):
|
|
106
|
+
p = Path(cookies)
|
|
107
|
+
if "\n" not in cookies and ";" not in cookies and "=" not in cookies and p.is_file():
|
|
108
|
+
return _parse_cookie_file(p) or None
|
|
109
|
+
return _parse_cookie_string(cookies) or None
|
|
110
|
+
return _filter_useful_cookies(cookies) or None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class _BaseClient:
|
|
114
|
+
"""Shared configuration, caching, and response validation logic."""
|
|
115
|
+
|
|
116
|
+
def __init__(
|
|
117
|
+
self,
|
|
118
|
+
*,
|
|
119
|
+
base_url: str = BASE_URL,
|
|
120
|
+
user_agent: str = DEFAULT_USER_AGENT,
|
|
121
|
+
cookies: dict[str, str] | str | Path | None = None,
|
|
122
|
+
rate_limit: float = 1.0,
|
|
123
|
+
max_retries: int = 3,
|
|
124
|
+
backoff_base: float = 1.0,
|
|
125
|
+
cache_ttl: float = 0.0,
|
|
126
|
+
respect_robots: bool = True,
|
|
127
|
+
raw_json: bool = True,
|
|
128
|
+
timeout: float = 20.0,
|
|
129
|
+
) -> None:
|
|
130
|
+
self.base_url = base_url.rstrip("/")
|
|
131
|
+
self.user_agent = user_agent
|
|
132
|
+
self.cookies: dict[str, str] | None = load_cookies(cookies)
|
|
133
|
+
self.min_interval = 1.0 / rate_limit if rate_limit > 0 else 0.0
|
|
134
|
+
self.max_retries = max_retries
|
|
135
|
+
self.backoff_base = backoff_base
|
|
136
|
+
self.cache_ttl = cache_ttl
|
|
137
|
+
self.respect_robots = respect_robots
|
|
138
|
+
self.raw_json = raw_json
|
|
139
|
+
self.timeout = timeout
|
|
140
|
+
|
|
141
|
+
self._cache: dict[str, tuple[float, httpx.Response]] = {}
|
|
142
|
+
self._disallow_patterns: list[re.Pattern[str]] = []
|
|
143
|
+
self._allow_patterns: list[re.Pattern[str]] = []
|
|
144
|
+
self._robots_loaded = False
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def has_cookies(self) -> bool:
|
|
148
|
+
"""Whether user-supplied cookies are configured on this client."""
|
|
149
|
+
return bool(self.cookies)
|
|
150
|
+
|
|
151
|
+
def format_result(self, result: Any) -> Any:
|
|
152
|
+
"""Return JSON-compatible dict when `raw_json=True`, otherwise the dataclass."""
|
|
153
|
+
if self.raw_json and hasattr(result, "to_dict"):
|
|
154
|
+
return result.to_dict()
|
|
155
|
+
return result
|
|
156
|
+
|
|
157
|
+
def _default_headers(self) -> dict[str, str]:
|
|
158
|
+
return {
|
|
159
|
+
"User-Agent": self.user_agent,
|
|
160
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
161
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
def _get_cached(self, url: str) -> httpx.Response | None:
|
|
165
|
+
if self.cache_ttl <= 0:
|
|
166
|
+
return None
|
|
167
|
+
entry = self._cache.get(url)
|
|
168
|
+
if entry is None:
|
|
169
|
+
return None
|
|
170
|
+
ts, resp = entry
|
|
171
|
+
if (time.monotonic() - ts) <= self.cache_ttl:
|
|
172
|
+
return resp
|
|
173
|
+
self._cache.pop(url, None)
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
def _set_cached(self, url: str, resp: httpx.Response) -> None:
|
|
177
|
+
if self.cache_ttl > 0:
|
|
178
|
+
self._cache[url] = (time.monotonic(), resp)
|
|
179
|
+
|
|
180
|
+
@staticmethod
|
|
181
|
+
def _compile_robots_pattern(pattern: str) -> re.Pattern[str]:
|
|
182
|
+
parts = [re.escape(p) for p in pattern.split("*")]
|
|
183
|
+
regex = "^" + ".*".join(parts)
|
|
184
|
+
if regex.endswith(r"\$"):
|
|
185
|
+
regex = regex[:-2] + "$"
|
|
186
|
+
return re.compile(regex)
|
|
187
|
+
|
|
188
|
+
def _load_robots_text(self, text: str) -> None:
|
|
189
|
+
disallow: list[re.Pattern[str]] = []
|
|
190
|
+
allow: list[re.Pattern[str]] = []
|
|
191
|
+
in_wildcard_group = False
|
|
192
|
+
for raw_line in text.splitlines():
|
|
193
|
+
line = raw_line.split("#", 1)[0].strip()
|
|
194
|
+
if not line or ":" not in line:
|
|
195
|
+
continue
|
|
196
|
+
key, val = (p.strip() for p in line.split(":", 1))
|
|
197
|
+
key_lower = key.lower()
|
|
198
|
+
if key_lower == "user-agent":
|
|
199
|
+
in_wildcard_group = val == "*"
|
|
200
|
+
elif in_wildcard_group and key_lower == "disallow" and val:
|
|
201
|
+
disallow.append(self._compile_robots_pattern(val))
|
|
202
|
+
elif in_wildcard_group and key_lower == "allow" and val:
|
|
203
|
+
allow.append(self._compile_robots_pattern(val))
|
|
204
|
+
self._disallow_patterns = disallow
|
|
205
|
+
self._allow_patterns = allow
|
|
206
|
+
self._robots_loaded = True
|
|
207
|
+
|
|
208
|
+
def _check_robots(self, url: str) -> None:
|
|
209
|
+
if not self.respect_robots or not self._robots_loaded:
|
|
210
|
+
return
|
|
211
|
+
parsed = urlparse(url)
|
|
212
|
+
path_with_query = parsed.path or "/"
|
|
213
|
+
if parsed.query:
|
|
214
|
+
path_with_query = f"{path_with_query}?{parsed.query}"
|
|
215
|
+
longest_allow = max(
|
|
216
|
+
(len(p.pattern) for p in self._allow_patterns if p.match(path_with_query)),
|
|
217
|
+
default=-1,
|
|
218
|
+
)
|
|
219
|
+
longest_disallow = max(
|
|
220
|
+
(len(p.pattern) for p in self._disallow_patterns if p.match(path_with_query)),
|
|
221
|
+
default=-1,
|
|
222
|
+
)
|
|
223
|
+
if longest_disallow >= 0 and longest_disallow > longest_allow:
|
|
224
|
+
raise RobotsDisallowedError(
|
|
225
|
+
f"URL disallowed by robots.txt for User-Agent {self.user_agent!r}: {url}"
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
def _validate_response(self, resp: httpx.Response, requested_url: str) -> httpx.Response:
|
|
229
|
+
final_url = str(resp.url)
|
|
230
|
+
if "/login" in final_url and "/login" not in requested_url:
|
|
231
|
+
raise NotSupportedError(
|
|
232
|
+
f"Last.fm redirected {requested_url} to login ({final_url}); "
|
|
233
|
+
"this endpoint/filter requires authentication."
|
|
234
|
+
)
|
|
235
|
+
if resp.status_code == 404:
|
|
236
|
+
raise NotFoundError(f"Resource not found (404): {requested_url}")
|
|
237
|
+
if resp.status_code in (429, 600):
|
|
238
|
+
if "Client Challenge" in resp.text or "/_fs-ch-" in resp.text:
|
|
239
|
+
raise RateLimitError(
|
|
240
|
+
f"Blocked by Last.fm anti-bot Client Challenge (HTTP {resp.status_code}): {requested_url}"
|
|
241
|
+
)
|
|
242
|
+
raise RateLimitError(
|
|
243
|
+
f"Rate limited by Last.fm (HTTP {resp.status_code}): {requested_url}"
|
|
244
|
+
)
|
|
245
|
+
if resp.status_code >= 500:
|
|
246
|
+
raise NetworkError(
|
|
247
|
+
f"Server error from Last.fm (HTTP {resp.status_code}): {requested_url}"
|
|
248
|
+
)
|
|
249
|
+
if resp.status_code >= 400:
|
|
250
|
+
raise NetworkError(
|
|
251
|
+
f"Unexpected HTTP {resp.status_code} for {requested_url}"
|
|
252
|
+
)
|
|
253
|
+
if "Client Challenge" in resp.text and "/_fs-ch-" in resp.text:
|
|
254
|
+
raise RateLimitError(
|
|
255
|
+
f"Blocked by Last.fm anti-bot Client Challenge: {requested_url}"
|
|
256
|
+
)
|
|
257
|
+
return resp
|
|
258
|
+
|
|
259
|
+
def _retry_delay(self, attempt: int, resp: httpx.Response | None = None) -> float:
|
|
260
|
+
if resp is not None:
|
|
261
|
+
retry_after = resp.headers.get("Retry-After")
|
|
262
|
+
if retry_after is not None:
|
|
263
|
+
try:
|
|
264
|
+
return max(0.0, float(retry_after))
|
|
265
|
+
except ValueError:
|
|
266
|
+
pass
|
|
267
|
+
return float(self.backoff_base * (2**attempt))
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class LastFmClient(_BaseClient):
|
|
271
|
+
"""Synchronous Last.fm HTML scraper client."""
|
|
272
|
+
|
|
273
|
+
def __init__(
|
|
274
|
+
self,
|
|
275
|
+
*,
|
|
276
|
+
transport: httpx.BaseTransport | None = None,
|
|
277
|
+
**kwargs: Any,
|
|
278
|
+
) -> None:
|
|
279
|
+
super().__init__(**kwargs)
|
|
280
|
+
self._client = httpx.Client(
|
|
281
|
+
base_url=self.base_url,
|
|
282
|
+
headers=self._default_headers(),
|
|
283
|
+
cookies=self.cookies,
|
|
284
|
+
follow_redirects=True,
|
|
285
|
+
timeout=self.timeout,
|
|
286
|
+
transport=transport,
|
|
287
|
+
)
|
|
288
|
+
self._last_request_at = 0.0
|
|
289
|
+
|
|
290
|
+
from fmscrape.api.album import AlbumAPI
|
|
291
|
+
from fmscrape.api.artist import ArtistAPI
|
|
292
|
+
from fmscrape.api.auth import AuthAPI
|
|
293
|
+
from fmscrape.api.chart import ChartAPI
|
|
294
|
+
from fmscrape.api.geo import GeoAPI
|
|
295
|
+
from fmscrape.api.library import LibraryAPI
|
|
296
|
+
from fmscrape.api.tag import TagAPI
|
|
297
|
+
from fmscrape.api.track import TrackAPI
|
|
298
|
+
from fmscrape.api.user import UserAPI
|
|
299
|
+
|
|
300
|
+
self.user = UserAPI(self)
|
|
301
|
+
self.artist = ArtistAPI(self)
|
|
302
|
+
self.album = AlbumAPI(self)
|
|
303
|
+
self.track = TrackAPI(self)
|
|
304
|
+
self.tag = TagAPI(self)
|
|
305
|
+
self.chart = ChartAPI(self)
|
|
306
|
+
self.geo = GeoAPI()
|
|
307
|
+
self.library = LibraryAPI(self)
|
|
308
|
+
self.auth = AuthAPI()
|
|
309
|
+
|
|
310
|
+
def close(self) -> None:
|
|
311
|
+
self._client.close()
|
|
312
|
+
|
|
313
|
+
def __enter__(self) -> LastFmClient:
|
|
314
|
+
return self
|
|
315
|
+
|
|
316
|
+
def __exit__(self, *args: object) -> None:
|
|
317
|
+
self.close()
|
|
318
|
+
|
|
319
|
+
def _wait_rate_limit(self) -> None:
|
|
320
|
+
if self.min_interval <= 0:
|
|
321
|
+
return
|
|
322
|
+
elapsed = time.monotonic() - self._last_request_at
|
|
323
|
+
if elapsed < self.min_interval:
|
|
324
|
+
time.sleep(self.min_interval - elapsed)
|
|
325
|
+
self._last_request_at = time.monotonic()
|
|
326
|
+
|
|
327
|
+
def _ensure_robots(self) -> None:
|
|
328
|
+
if not self.respect_robots or self._robots_loaded:
|
|
329
|
+
return
|
|
330
|
+
try:
|
|
331
|
+
self._wait_rate_limit()
|
|
332
|
+
resp = self._client.get("/robots.txt")
|
|
333
|
+
if resp.status_code == 200:
|
|
334
|
+
self._load_robots_text(resp.text)
|
|
335
|
+
else:
|
|
336
|
+
self._robots_loaded = True
|
|
337
|
+
except httpx.HTTPError:
|
|
338
|
+
self._robots_loaded = True
|
|
339
|
+
|
|
340
|
+
def get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response:
|
|
341
|
+
url = _build_url(path, params)
|
|
342
|
+
cached = self._get_cached(url)
|
|
343
|
+
if cached is not None:
|
|
344
|
+
return cached
|
|
345
|
+
|
|
346
|
+
self._ensure_robots()
|
|
347
|
+
self._check_robots(url)
|
|
348
|
+
|
|
349
|
+
last_exc: Exception | None = None
|
|
350
|
+
for attempt in range(self.max_retries + 1):
|
|
351
|
+
self._wait_rate_limit()
|
|
352
|
+
try:
|
|
353
|
+
resp = self._client.get(path, params={k: v for k, v in (params or {}).items() if v is not None})
|
|
354
|
+
except httpx.HTTPError as exc:
|
|
355
|
+
last_exc = exc
|
|
356
|
+
if attempt < self.max_retries:
|
|
357
|
+
time.sleep(self._retry_delay(attempt))
|
|
358
|
+
continue
|
|
359
|
+
raise NetworkError(f"Network error fetching {url}: {exc}") from exc
|
|
360
|
+
|
|
361
|
+
if resp.status_code in _RETRYABLE_STATUSES and attempt < self.max_retries:
|
|
362
|
+
time.sleep(self._retry_delay(attempt, resp))
|
|
363
|
+
continue
|
|
364
|
+
|
|
365
|
+
validated = self._validate_response(resp, url)
|
|
366
|
+
self._set_cached(url, validated)
|
|
367
|
+
return validated
|
|
368
|
+
|
|
369
|
+
raise NetworkError(f"Failed to fetch {url}: {last_exc}")
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
class AsyncLastFmClient(_BaseClient):
|
|
373
|
+
"""Asynchronous Last.fm HTML scraper client."""
|
|
374
|
+
|
|
375
|
+
def __init__(
|
|
376
|
+
self,
|
|
377
|
+
*,
|
|
378
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
379
|
+
**kwargs: Any,
|
|
380
|
+
) -> None:
|
|
381
|
+
super().__init__(**kwargs)
|
|
382
|
+
self._client = httpx.AsyncClient(
|
|
383
|
+
base_url=self.base_url,
|
|
384
|
+
headers=self._default_headers(),
|
|
385
|
+
cookies=self.cookies,
|
|
386
|
+
follow_redirects=True,
|
|
387
|
+
timeout=self.timeout,
|
|
388
|
+
transport=transport,
|
|
389
|
+
)
|
|
390
|
+
self._last_request_at = 0.0
|
|
391
|
+
self._lock = asyncio.Lock()
|
|
392
|
+
|
|
393
|
+
from fmscrape.api.album import AsyncAlbumAPI
|
|
394
|
+
from fmscrape.api.artist import AsyncArtistAPI
|
|
395
|
+
from fmscrape.api.auth import AuthAPI
|
|
396
|
+
from fmscrape.api.chart import AsyncChartAPI
|
|
397
|
+
from fmscrape.api.geo import GeoAPI
|
|
398
|
+
from fmscrape.api.library import AsyncLibraryAPI
|
|
399
|
+
from fmscrape.api.tag import AsyncTagAPI
|
|
400
|
+
from fmscrape.api.track import AsyncTrackAPI
|
|
401
|
+
from fmscrape.api.user import AsyncUserAPI
|
|
402
|
+
|
|
403
|
+
self.user = AsyncUserAPI(self)
|
|
404
|
+
self.artist = AsyncArtistAPI(self)
|
|
405
|
+
self.album = AsyncAlbumAPI(self)
|
|
406
|
+
self.track = AsyncTrackAPI(self)
|
|
407
|
+
self.tag = AsyncTagAPI(self)
|
|
408
|
+
self.chart = AsyncChartAPI(self)
|
|
409
|
+
self.geo = GeoAPI()
|
|
410
|
+
self.library = AsyncLibraryAPI(self)
|
|
411
|
+
self.auth = AuthAPI()
|
|
412
|
+
|
|
413
|
+
async def aclose(self) -> None:
|
|
414
|
+
await self._client.aclose()
|
|
415
|
+
|
|
416
|
+
async def __aenter__(self) -> AsyncLastFmClient:
|
|
417
|
+
return self
|
|
418
|
+
|
|
419
|
+
async def __aexit__(self, *args: object) -> None:
|
|
420
|
+
await self.aclose()
|
|
421
|
+
|
|
422
|
+
async def _wait_rate_limit(self) -> None:
|
|
423
|
+
if self.min_interval <= 0:
|
|
424
|
+
return
|
|
425
|
+
async with self._lock:
|
|
426
|
+
elapsed = time.monotonic() - self._last_request_at
|
|
427
|
+
if elapsed < self.min_interval:
|
|
428
|
+
await asyncio.sleep(self.min_interval - elapsed)
|
|
429
|
+
self._last_request_at = time.monotonic()
|
|
430
|
+
|
|
431
|
+
async def _ensure_robots(self) -> None:
|
|
432
|
+
if not self.respect_robots or self._robots_loaded:
|
|
433
|
+
return
|
|
434
|
+
try:
|
|
435
|
+
await self._wait_rate_limit()
|
|
436
|
+
resp = await self._client.get("/robots.txt")
|
|
437
|
+
if resp.status_code == 200:
|
|
438
|
+
self._load_robots_text(resp.text)
|
|
439
|
+
else:
|
|
440
|
+
self._robots_loaded = True
|
|
441
|
+
except httpx.HTTPError:
|
|
442
|
+
self._robots_loaded = True
|
|
443
|
+
|
|
444
|
+
async def get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response:
|
|
445
|
+
url = _build_url(path, params)
|
|
446
|
+
cached = self._get_cached(url)
|
|
447
|
+
if cached is not None:
|
|
448
|
+
return cached
|
|
449
|
+
|
|
450
|
+
await self._ensure_robots()
|
|
451
|
+
self._check_robots(url)
|
|
452
|
+
|
|
453
|
+
last_exc: Exception | None = None
|
|
454
|
+
for attempt in range(self.max_retries + 1):
|
|
455
|
+
await self._wait_rate_limit()
|
|
456
|
+
try:
|
|
457
|
+
resp = await self._client.get(path, params={k: v for k, v in (params or {}).items() if v is not None})
|
|
458
|
+
except httpx.HTTPError as exc:
|
|
459
|
+
last_exc = exc
|
|
460
|
+
if attempt < self.max_retries:
|
|
461
|
+
await asyncio.sleep(self._retry_delay(attempt))
|
|
462
|
+
continue
|
|
463
|
+
raise NetworkError(f"Network error fetching {url}: {exc}") from exc
|
|
464
|
+
|
|
465
|
+
if resp.status_code in _RETRYABLE_STATUSES and attempt < self.max_retries:
|
|
466
|
+
await asyncio.sleep(self._retry_delay(attempt, resp))
|
|
467
|
+
continue
|
|
468
|
+
|
|
469
|
+
validated = self._validate_response(resp, url)
|
|
470
|
+
self._set_cached(url, validated)
|
|
471
|
+
return validated
|
|
472
|
+
|
|
473
|
+
raise NetworkError(f"Failed to fetch {url}: {last_exc}")
|
|
474
|
+
|
fmscrape/errors.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Custom exceptions for fmscrape."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class FmScrapeError(Exception):
|
|
7
|
+
"""Base exception for all fmscrape errors."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NotFoundError(FmScrapeError):
|
|
11
|
+
"""Raised when the requested Last.fm resource does not exist (HTTP 404)."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RateLimitError(FmScrapeError):
|
|
15
|
+
"""Raised when Last.fm rate-limits requests (HTTP 429 or 600) after retries."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class NetworkError(FmScrapeError):
|
|
19
|
+
"""Raised on network/transport failures or unexpected 5xx responses after retries."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ParseError(FmScrapeError):
|
|
23
|
+
"""Raised when expected HTML structure or selector cannot be parsed."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, message: str, *, url: str, selector: str) -> None:
|
|
26
|
+
self.url = url
|
|
27
|
+
self.selector = selector
|
|
28
|
+
super().__init__(f"{message} (url={url!r}, selector={selector!r})")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class NotSupportedError(FmScrapeError):
|
|
32
|
+
"""Raised for methods requiring authentication or unavailable on public pages."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class RobotsDisallowedError(FmScrapeError):
|
|
36
|
+
"""Raised when robots.txt disallows fetching the target URL."""
|
|
37
|
+
|