fmscrape 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of fmscrape might be problematic. Click here for more details.

fmscrape/client.py ADDED
@@ -0,0 +1,474 @@
1
+ """HTTP client with rate limiting, robots.txt compliance, TTL cache, and exponential backoff."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import json
7
+ import re
8
+ import time
9
+ from pathlib import Path
10
+ from typing import Any
11
+ from urllib.parse import urlencode, urljoin, urlparse
12
+
13
+ import httpx
14
+
15
+ from fmscrape.errors import (
16
+ NetworkError,
17
+ NotFoundError,
18
+ NotSupportedError,
19
+ RateLimitError,
20
+ RobotsDisallowedError,
21
+ )
22
+ from fmscrape.normalize import BASE_URL
23
+
24
+ DEFAULT_USER_AGENT = (
25
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
26
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
27
+ "Chrome/131.0.0.0 Safari/537.36"
28
+ )
29
+
30
+ _RETRYABLE_STATUSES = {429, 500, 502, 503, 504, 600}
31
+
32
+
33
+ def _build_url(path: str, params: dict[str, Any] | None = None) -> str:
34
+ base = path if path.startswith(("http://", "https://")) else urljoin(BASE_URL, path)
35
+ if not params:
36
+ return base
37
+ clean_params = {k: v for k, v in params.items() if v is not None}
38
+ if not clean_params:
39
+ return base
40
+ sep = "&" if "?" in base else "?"
41
+ return f"{base}{sep}{urlencode(clean_params)}"
42
+
43
+
44
+ _USEFUL_COOKIES = frozenset({"sessionid", "csrftoken", "fsuid", "lfmanon"})
45
+
46
+
47
+ def _filter_useful_cookies(cookies: dict[str, str]) -> dict[str, str]:
48
+ useful = {k: v for k, v in cookies.items() if k in _USEFUL_COOKIES}
49
+ return useful if useful else cookies
50
+
51
+
52
+ def _parse_cookie_string(raw: str) -> dict[str, str]:
53
+ cookies: dict[str, str] = {}
54
+ for part in raw.replace("\n", ";").split(";"):
55
+ if "=" not in part:
56
+ continue
57
+ k, v = part.split("=", 1)
58
+ key = k.strip()
59
+ if key:
60
+ cookies[key] = v.strip().strip("'\"")
61
+ return _filter_useful_cookies(cookies)
62
+
63
+
64
+ def _parse_cookie_file(path: Path) -> dict[str, str]:
65
+ text = path.read_text(encoding="utf-8").strip()
66
+ if not text:
67
+ return {}
68
+ if text.startswith(("{", "[")):
69
+ data = json.loads(text)
70
+ if isinstance(data, dict):
71
+ return _filter_useful_cookies({str(k): str(v) for k, v in data.items()})
72
+ if isinstance(data, list):
73
+ cookies = {
74
+ str(item["name"]): str(item["value"])
75
+ for item in data
76
+ if isinstance(item, dict) and "name" in item and "value" in item
77
+ }
78
+ return _filter_useful_cookies(cookies)
79
+
80
+ # Netscape cookies.txt (7 tab-separated columns) or key=value lines
81
+ netscape: dict[str, str] = {}
82
+ for line in text.splitlines():
83
+ line = line.strip()
84
+ if not line or line.startswith("#"):
85
+ continue
86
+ cols = line.split("\t")
87
+ if len(cols) >= 7:
88
+ netscape[cols[5].strip()] = cols[6].strip()
89
+ if netscape:
90
+ return _filter_useful_cookies(netscape)
91
+ return _parse_cookie_string(text)
92
+
93
+
94
+ def load_cookies(cookies: dict[str, str] | str | Path | None) -> dict[str, str] | None:
95
+ """Resolve cookies from dict, raw header string, file path, or cwd cookies.txt/cookies.json."""
96
+ if cookies is None:
97
+ for candidate in ("cookies.txt", "cookies.json"):
98
+ p = Path.cwd() / candidate
99
+ if p.is_file():
100
+ parsed = _parse_cookie_file(p)
101
+ return parsed or None
102
+ return None
103
+ if isinstance(cookies, Path):
104
+ return _parse_cookie_file(cookies) or None
105
+ if isinstance(cookies, str):
106
+ p = Path(cookies)
107
+ if "\n" not in cookies and ";" not in cookies and "=" not in cookies and p.is_file():
108
+ return _parse_cookie_file(p) or None
109
+ return _parse_cookie_string(cookies) or None
110
+ return _filter_useful_cookies(cookies) or None
111
+
112
+
113
+ class _BaseClient:
114
+ """Shared configuration, caching, and response validation logic."""
115
+
116
+ def __init__(
117
+ self,
118
+ *,
119
+ base_url: str = BASE_URL,
120
+ user_agent: str = DEFAULT_USER_AGENT,
121
+ cookies: dict[str, str] | str | Path | None = None,
122
+ rate_limit: float = 1.0,
123
+ max_retries: int = 3,
124
+ backoff_base: float = 1.0,
125
+ cache_ttl: float = 0.0,
126
+ respect_robots: bool = True,
127
+ raw_json: bool = True,
128
+ timeout: float = 20.0,
129
+ ) -> None:
130
+ self.base_url = base_url.rstrip("/")
131
+ self.user_agent = user_agent
132
+ self.cookies: dict[str, str] | None = load_cookies(cookies)
133
+ self.min_interval = 1.0 / rate_limit if rate_limit > 0 else 0.0
134
+ self.max_retries = max_retries
135
+ self.backoff_base = backoff_base
136
+ self.cache_ttl = cache_ttl
137
+ self.respect_robots = respect_robots
138
+ self.raw_json = raw_json
139
+ self.timeout = timeout
140
+
141
+ self._cache: dict[str, tuple[float, httpx.Response]] = {}
142
+ self._disallow_patterns: list[re.Pattern[str]] = []
143
+ self._allow_patterns: list[re.Pattern[str]] = []
144
+ self._robots_loaded = False
145
+
146
+ @property
147
+ def has_cookies(self) -> bool:
148
+ """Whether user-supplied cookies are configured on this client."""
149
+ return bool(self.cookies)
150
+
151
+ def format_result(self, result: Any) -> Any:
152
+ """Return JSON-compatible dict when `raw_json=True`, otherwise the dataclass."""
153
+ if self.raw_json and hasattr(result, "to_dict"):
154
+ return result.to_dict()
155
+ return result
156
+
157
+ def _default_headers(self) -> dict[str, str]:
158
+ return {
159
+ "User-Agent": self.user_agent,
160
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
161
+ "Accept-Language": "en-US,en;q=0.9",
162
+ }
163
+
164
+ def _get_cached(self, url: str) -> httpx.Response | None:
165
+ if self.cache_ttl <= 0:
166
+ return None
167
+ entry = self._cache.get(url)
168
+ if entry is None:
169
+ return None
170
+ ts, resp = entry
171
+ if (time.monotonic() - ts) <= self.cache_ttl:
172
+ return resp
173
+ self._cache.pop(url, None)
174
+ return None
175
+
176
+ def _set_cached(self, url: str, resp: httpx.Response) -> None:
177
+ if self.cache_ttl > 0:
178
+ self._cache[url] = (time.monotonic(), resp)
179
+
180
+ @staticmethod
181
+ def _compile_robots_pattern(pattern: str) -> re.Pattern[str]:
182
+ parts = [re.escape(p) for p in pattern.split("*")]
183
+ regex = "^" + ".*".join(parts)
184
+ if regex.endswith(r"\$"):
185
+ regex = regex[:-2] + "$"
186
+ return re.compile(regex)
187
+
188
+ def _load_robots_text(self, text: str) -> None:
189
+ disallow: list[re.Pattern[str]] = []
190
+ allow: list[re.Pattern[str]] = []
191
+ in_wildcard_group = False
192
+ for raw_line in text.splitlines():
193
+ line = raw_line.split("#", 1)[0].strip()
194
+ if not line or ":" not in line:
195
+ continue
196
+ key, val = (p.strip() for p in line.split(":", 1))
197
+ key_lower = key.lower()
198
+ if key_lower == "user-agent":
199
+ in_wildcard_group = val == "*"
200
+ elif in_wildcard_group and key_lower == "disallow" and val:
201
+ disallow.append(self._compile_robots_pattern(val))
202
+ elif in_wildcard_group and key_lower == "allow" and val:
203
+ allow.append(self._compile_robots_pattern(val))
204
+ self._disallow_patterns = disallow
205
+ self._allow_patterns = allow
206
+ self._robots_loaded = True
207
+
208
+ def _check_robots(self, url: str) -> None:
209
+ if not self.respect_robots or not self._robots_loaded:
210
+ return
211
+ parsed = urlparse(url)
212
+ path_with_query = parsed.path or "/"
213
+ if parsed.query:
214
+ path_with_query = f"{path_with_query}?{parsed.query}"
215
+ longest_allow = max(
216
+ (len(p.pattern) for p in self._allow_patterns if p.match(path_with_query)),
217
+ default=-1,
218
+ )
219
+ longest_disallow = max(
220
+ (len(p.pattern) for p in self._disallow_patterns if p.match(path_with_query)),
221
+ default=-1,
222
+ )
223
+ if longest_disallow >= 0 and longest_disallow > longest_allow:
224
+ raise RobotsDisallowedError(
225
+ f"URL disallowed by robots.txt for User-Agent {self.user_agent!r}: {url}"
226
+ )
227
+
228
+ def _validate_response(self, resp: httpx.Response, requested_url: str) -> httpx.Response:
229
+ final_url = str(resp.url)
230
+ if "/login" in final_url and "/login" not in requested_url:
231
+ raise NotSupportedError(
232
+ f"Last.fm redirected {requested_url} to login ({final_url}); "
233
+ "this endpoint/filter requires authentication."
234
+ )
235
+ if resp.status_code == 404:
236
+ raise NotFoundError(f"Resource not found (404): {requested_url}")
237
+ if resp.status_code in (429, 600):
238
+ if "Client Challenge" in resp.text or "/_fs-ch-" in resp.text:
239
+ raise RateLimitError(
240
+ f"Blocked by Last.fm anti-bot Client Challenge (HTTP {resp.status_code}): {requested_url}"
241
+ )
242
+ raise RateLimitError(
243
+ f"Rate limited by Last.fm (HTTP {resp.status_code}): {requested_url}"
244
+ )
245
+ if resp.status_code >= 500:
246
+ raise NetworkError(
247
+ f"Server error from Last.fm (HTTP {resp.status_code}): {requested_url}"
248
+ )
249
+ if resp.status_code >= 400:
250
+ raise NetworkError(
251
+ f"Unexpected HTTP {resp.status_code} for {requested_url}"
252
+ )
253
+ if "Client Challenge" in resp.text and "/_fs-ch-" in resp.text:
254
+ raise RateLimitError(
255
+ f"Blocked by Last.fm anti-bot Client Challenge: {requested_url}"
256
+ )
257
+ return resp
258
+
259
+ def _retry_delay(self, attempt: int, resp: httpx.Response | None = None) -> float:
260
+ if resp is not None:
261
+ retry_after = resp.headers.get("Retry-After")
262
+ if retry_after is not None:
263
+ try:
264
+ return max(0.0, float(retry_after))
265
+ except ValueError:
266
+ pass
267
+ return float(self.backoff_base * (2**attempt))
268
+
269
+
270
+ class LastFmClient(_BaseClient):
271
+ """Synchronous Last.fm HTML scraper client."""
272
+
273
+ def __init__(
274
+ self,
275
+ *,
276
+ transport: httpx.BaseTransport | None = None,
277
+ **kwargs: Any,
278
+ ) -> None:
279
+ super().__init__(**kwargs)
280
+ self._client = httpx.Client(
281
+ base_url=self.base_url,
282
+ headers=self._default_headers(),
283
+ cookies=self.cookies,
284
+ follow_redirects=True,
285
+ timeout=self.timeout,
286
+ transport=transport,
287
+ )
288
+ self._last_request_at = 0.0
289
+
290
+ from fmscrape.api.album import AlbumAPI
291
+ from fmscrape.api.artist import ArtistAPI
292
+ from fmscrape.api.auth import AuthAPI
293
+ from fmscrape.api.chart import ChartAPI
294
+ from fmscrape.api.geo import GeoAPI
295
+ from fmscrape.api.library import LibraryAPI
296
+ from fmscrape.api.tag import TagAPI
297
+ from fmscrape.api.track import TrackAPI
298
+ from fmscrape.api.user import UserAPI
299
+
300
+ self.user = UserAPI(self)
301
+ self.artist = ArtistAPI(self)
302
+ self.album = AlbumAPI(self)
303
+ self.track = TrackAPI(self)
304
+ self.tag = TagAPI(self)
305
+ self.chart = ChartAPI(self)
306
+ self.geo = GeoAPI()
307
+ self.library = LibraryAPI(self)
308
+ self.auth = AuthAPI()
309
+
310
+ def close(self) -> None:
311
+ self._client.close()
312
+
313
+ def __enter__(self) -> LastFmClient:
314
+ return self
315
+
316
+ def __exit__(self, *args: object) -> None:
317
+ self.close()
318
+
319
+ def _wait_rate_limit(self) -> None:
320
+ if self.min_interval <= 0:
321
+ return
322
+ elapsed = time.monotonic() - self._last_request_at
323
+ if elapsed < self.min_interval:
324
+ time.sleep(self.min_interval - elapsed)
325
+ self._last_request_at = time.monotonic()
326
+
327
+ def _ensure_robots(self) -> None:
328
+ if not self.respect_robots or self._robots_loaded:
329
+ return
330
+ try:
331
+ self._wait_rate_limit()
332
+ resp = self._client.get("/robots.txt")
333
+ if resp.status_code == 200:
334
+ self._load_robots_text(resp.text)
335
+ else:
336
+ self._robots_loaded = True
337
+ except httpx.HTTPError:
338
+ self._robots_loaded = True
339
+
340
+ def get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response:
341
+ url = _build_url(path, params)
342
+ cached = self._get_cached(url)
343
+ if cached is not None:
344
+ return cached
345
+
346
+ self._ensure_robots()
347
+ self._check_robots(url)
348
+
349
+ last_exc: Exception | None = None
350
+ for attempt in range(self.max_retries + 1):
351
+ self._wait_rate_limit()
352
+ try:
353
+ resp = self._client.get(path, params={k: v for k, v in (params or {}).items() if v is not None})
354
+ except httpx.HTTPError as exc:
355
+ last_exc = exc
356
+ if attempt < self.max_retries:
357
+ time.sleep(self._retry_delay(attempt))
358
+ continue
359
+ raise NetworkError(f"Network error fetching {url}: {exc}") from exc
360
+
361
+ if resp.status_code in _RETRYABLE_STATUSES and attempt < self.max_retries:
362
+ time.sleep(self._retry_delay(attempt, resp))
363
+ continue
364
+
365
+ validated = self._validate_response(resp, url)
366
+ self._set_cached(url, validated)
367
+ return validated
368
+
369
+ raise NetworkError(f"Failed to fetch {url}: {last_exc}")
370
+
371
+
372
+ class AsyncLastFmClient(_BaseClient):
373
+ """Asynchronous Last.fm HTML scraper client."""
374
+
375
+ def __init__(
376
+ self,
377
+ *,
378
+ transport: httpx.AsyncBaseTransport | None = None,
379
+ **kwargs: Any,
380
+ ) -> None:
381
+ super().__init__(**kwargs)
382
+ self._client = httpx.AsyncClient(
383
+ base_url=self.base_url,
384
+ headers=self._default_headers(),
385
+ cookies=self.cookies,
386
+ follow_redirects=True,
387
+ timeout=self.timeout,
388
+ transport=transport,
389
+ )
390
+ self._last_request_at = 0.0
391
+ self._lock = asyncio.Lock()
392
+
393
+ from fmscrape.api.album import AsyncAlbumAPI
394
+ from fmscrape.api.artist import AsyncArtistAPI
395
+ from fmscrape.api.auth import AuthAPI
396
+ from fmscrape.api.chart import AsyncChartAPI
397
+ from fmscrape.api.geo import GeoAPI
398
+ from fmscrape.api.library import AsyncLibraryAPI
399
+ from fmscrape.api.tag import AsyncTagAPI
400
+ from fmscrape.api.track import AsyncTrackAPI
401
+ from fmscrape.api.user import AsyncUserAPI
402
+
403
+ self.user = AsyncUserAPI(self)
404
+ self.artist = AsyncArtistAPI(self)
405
+ self.album = AsyncAlbumAPI(self)
406
+ self.track = AsyncTrackAPI(self)
407
+ self.tag = AsyncTagAPI(self)
408
+ self.chart = AsyncChartAPI(self)
409
+ self.geo = GeoAPI()
410
+ self.library = AsyncLibraryAPI(self)
411
+ self.auth = AuthAPI()
412
+
413
+ async def aclose(self) -> None:
414
+ await self._client.aclose()
415
+
416
+ async def __aenter__(self) -> AsyncLastFmClient:
417
+ return self
418
+
419
+ async def __aexit__(self, *args: object) -> None:
420
+ await self.aclose()
421
+
422
+ async def _wait_rate_limit(self) -> None:
423
+ if self.min_interval <= 0:
424
+ return
425
+ async with self._lock:
426
+ elapsed = time.monotonic() - self._last_request_at
427
+ if elapsed < self.min_interval:
428
+ await asyncio.sleep(self.min_interval - elapsed)
429
+ self._last_request_at = time.monotonic()
430
+
431
+ async def _ensure_robots(self) -> None:
432
+ if not self.respect_robots or self._robots_loaded:
433
+ return
434
+ try:
435
+ await self._wait_rate_limit()
436
+ resp = await self._client.get("/robots.txt")
437
+ if resp.status_code == 200:
438
+ self._load_robots_text(resp.text)
439
+ else:
440
+ self._robots_loaded = True
441
+ except httpx.HTTPError:
442
+ self._robots_loaded = True
443
+
444
+ async def get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response:
445
+ url = _build_url(path, params)
446
+ cached = self._get_cached(url)
447
+ if cached is not None:
448
+ return cached
449
+
450
+ await self._ensure_robots()
451
+ self._check_robots(url)
452
+
453
+ last_exc: Exception | None = None
454
+ for attempt in range(self.max_retries + 1):
455
+ await self._wait_rate_limit()
456
+ try:
457
+ resp = await self._client.get(path, params={k: v for k, v in (params or {}).items() if v is not None})
458
+ except httpx.HTTPError as exc:
459
+ last_exc = exc
460
+ if attempt < self.max_retries:
461
+ await asyncio.sleep(self._retry_delay(attempt))
462
+ continue
463
+ raise NetworkError(f"Network error fetching {url}: {exc}") from exc
464
+
465
+ if resp.status_code in _RETRYABLE_STATUSES and attempt < self.max_retries:
466
+ await asyncio.sleep(self._retry_delay(attempt, resp))
467
+ continue
468
+
469
+ validated = self._validate_response(resp, url)
470
+ self._set_cached(url, validated)
471
+ return validated
472
+
473
+ raise NetworkError(f"Failed to fetch {url}: {last_exc}")
474
+
fmscrape/errors.py ADDED
@@ -0,0 +1,37 @@
1
+ """Custom exceptions for fmscrape."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class FmScrapeError(Exception):
7
+ """Base exception for all fmscrape errors."""
8
+
9
+
10
+ class NotFoundError(FmScrapeError):
11
+ """Raised when the requested Last.fm resource does not exist (HTTP 404)."""
12
+
13
+
14
+ class RateLimitError(FmScrapeError):
15
+ """Raised when Last.fm rate-limits requests (HTTP 429 or 600) after retries."""
16
+
17
+
18
+ class NetworkError(FmScrapeError):
19
+ """Raised on network/transport failures or unexpected 5xx responses after retries."""
20
+
21
+
22
+ class ParseError(FmScrapeError):
23
+ """Raised when expected HTML structure or selector cannot be parsed."""
24
+
25
+ def __init__(self, message: str, *, url: str, selector: str) -> None:
26
+ self.url = url
27
+ self.selector = selector
28
+ super().__init__(f"{message} (url={url!r}, selector={selector!r})")
29
+
30
+
31
+ class NotSupportedError(FmScrapeError):
32
+ """Raised for methods requiring authentication or unavailable on public pages."""
33
+
34
+
35
+ class RobotsDisallowedError(FmScrapeError):
36
+ """Raised when robots.txt disallows fetching the target URL."""
37
+