quasarr 2.4.8__py3-none-any.whl → 2.4.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of quasarr might be problematic. Click here for more details.
- quasarr/__init__.py +134 -70
- quasarr/api/__init__.py +40 -31
- quasarr/api/arr/__init__.py +116 -108
- quasarr/api/captcha/__init__.py +262 -137
- quasarr/api/config/__init__.py +76 -46
- quasarr/api/packages/__init__.py +138 -102
- quasarr/api/sponsors_helper/__init__.py +29 -16
- quasarr/api/statistics/__init__.py +19 -19
- quasarr/downloads/__init__.py +165 -72
- quasarr/downloads/linkcrypters/al.py +35 -18
- quasarr/downloads/linkcrypters/filecrypt.py +107 -52
- quasarr/downloads/linkcrypters/hide.py +5 -6
- quasarr/downloads/packages/__init__.py +342 -177
- quasarr/downloads/sources/al.py +191 -100
- quasarr/downloads/sources/by.py +31 -13
- quasarr/downloads/sources/dd.py +27 -14
- quasarr/downloads/sources/dj.py +1 -3
- quasarr/downloads/sources/dl.py +126 -71
- quasarr/downloads/sources/dt.py +11 -5
- quasarr/downloads/sources/dw.py +28 -14
- quasarr/downloads/sources/he.py +32 -24
- quasarr/downloads/sources/mb.py +19 -9
- quasarr/downloads/sources/nk.py +14 -10
- quasarr/downloads/sources/nx.py +8 -18
- quasarr/downloads/sources/sf.py +45 -20
- quasarr/downloads/sources/sj.py +1 -3
- quasarr/downloads/sources/sl.py +9 -5
- quasarr/downloads/sources/wd.py +32 -12
- quasarr/downloads/sources/wx.py +35 -21
- quasarr/providers/auth.py +42 -37
- quasarr/providers/cloudflare.py +28 -30
- quasarr/providers/hostname_issues.py +2 -1
- quasarr/providers/html_images.py +2 -2
- quasarr/providers/html_templates.py +22 -14
- quasarr/providers/imdb_metadata.py +149 -80
- quasarr/providers/jd_cache.py +131 -39
- quasarr/providers/log.py +1 -1
- quasarr/providers/myjd_api.py +260 -196
- quasarr/providers/notifications.py +53 -41
- quasarr/providers/obfuscated.py +9 -4
- quasarr/providers/sessions/al.py +71 -55
- quasarr/providers/sessions/dd.py +21 -14
- quasarr/providers/sessions/dl.py +30 -19
- quasarr/providers/sessions/nx.py +23 -14
- quasarr/providers/shared_state.py +292 -141
- quasarr/providers/statistics.py +75 -43
- quasarr/providers/utils.py +33 -27
- quasarr/providers/version.py +45 -14
- quasarr/providers/web_server.py +10 -5
- quasarr/search/__init__.py +30 -18
- quasarr/search/sources/al.py +124 -73
- quasarr/search/sources/by.py +110 -59
- quasarr/search/sources/dd.py +57 -35
- quasarr/search/sources/dj.py +69 -48
- quasarr/search/sources/dl.py +159 -100
- quasarr/search/sources/dt.py +110 -74
- quasarr/search/sources/dw.py +121 -61
- quasarr/search/sources/fx.py +108 -62
- quasarr/search/sources/he.py +78 -49
- quasarr/search/sources/mb.py +96 -48
- quasarr/search/sources/nk.py +80 -50
- quasarr/search/sources/nx.py +91 -62
- quasarr/search/sources/sf.py +171 -106
- quasarr/search/sources/sj.py +69 -48
- quasarr/search/sources/sl.py +115 -71
- quasarr/search/sources/wd.py +67 -44
- quasarr/search/sources/wx.py +188 -123
- quasarr/storage/config.py +65 -52
- quasarr/storage/setup.py +238 -140
- quasarr/storage/sqlite_database.py +10 -4
- {quasarr-2.4.8.dist-info → quasarr-2.4.10.dist-info}/METADATA +4 -3
- quasarr-2.4.10.dist-info/RECORD +81 -0
- quasarr-2.4.8.dist-info/RECORD +0 -81
- {quasarr-2.4.8.dist-info → quasarr-2.4.10.dist-info}/WHEEL +0 -0
- {quasarr-2.4.8.dist-info → quasarr-2.4.10.dist-info}/entry_points.txt +0 -0
- {quasarr-2.4.8.dist-info → quasarr-2.4.10.dist-info}/licenses/LICENSE +0 -0
quasarr/search/sources/al.py
CHANGED
|
@@ -5,16 +5,19 @@
|
|
|
5
5
|
import time
|
|
6
6
|
from base64 import urlsafe_b64encode
|
|
7
7
|
from html import unescape
|
|
8
|
-
from urllib.parse import
|
|
8
|
+
from urllib.parse import quote_plus, urljoin
|
|
9
9
|
|
|
10
10
|
from bs4 import BeautifulSoup
|
|
11
11
|
|
|
12
|
-
from quasarr.downloads.sources.al import (
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
from quasarr.downloads.sources.al import (
|
|
13
|
+
guess_title,
|
|
14
|
+
parse_info_from_download_item,
|
|
15
|
+
parse_info_from_feed_entry,
|
|
16
|
+
)
|
|
17
|
+
from quasarr.providers.hostname_issues import clear_hostname_issue, mark_hostname_issue
|
|
15
18
|
from quasarr.providers.imdb_metadata import get_localized_title
|
|
16
|
-
from quasarr.providers.log import
|
|
17
|
-
from quasarr.providers.sessions.al import
|
|
19
|
+
from quasarr.providers.log import debug, info
|
|
20
|
+
from quasarr.providers.sessions.al import fetch_via_requests_session, invalidate_session
|
|
18
21
|
|
|
19
22
|
hostname = "al"
|
|
20
23
|
supported_mirrors = ["rapidgator", "ddownload"]
|
|
@@ -77,7 +80,7 @@ def parse_relative_date(raw: str) -> datetime | None:
|
|
|
77
80
|
unit = english_match.group(2).lower()
|
|
78
81
|
|
|
79
82
|
# Remove plural 's' if present
|
|
80
|
-
if unit.endswith(
|
|
83
|
+
if unit.endswith("s"):
|
|
81
84
|
unit = unit[:-1]
|
|
82
85
|
|
|
83
86
|
if unit.startswith("second"):
|
|
@@ -124,7 +127,9 @@ def al_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
124
127
|
host = shared_state.values["config"]("Hostnames").get(hostname)
|
|
125
128
|
|
|
126
129
|
if not "arr" in request_from.lower():
|
|
127
|
-
debug(
|
|
130
|
+
debug(
|
|
131
|
+
f'Skipping {request_from} search on "{hostname.upper()}" (unsupported media type)!'
|
|
132
|
+
)
|
|
128
133
|
return releases
|
|
129
134
|
|
|
130
135
|
if "Radarr" in request_from:
|
|
@@ -137,15 +142,19 @@ def al_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
137
142
|
return releases
|
|
138
143
|
|
|
139
144
|
try:
|
|
140
|
-
r = fetch_via_requests_session(
|
|
145
|
+
r = fetch_via_requests_session(
|
|
146
|
+
shared_state, method="GET", target_url=f"https://www.{host}/", timeout=30
|
|
147
|
+
)
|
|
141
148
|
r.raise_for_status()
|
|
142
149
|
except Exception as e:
|
|
143
150
|
info(f"{hostname}: could not fetch feed: {e}")
|
|
144
|
-
mark_hostname_issue(
|
|
151
|
+
mark_hostname_issue(
|
|
152
|
+
hostname, "feed", str(e) if "e" in dir() else "Error occurred"
|
|
153
|
+
)
|
|
145
154
|
invalidate_session(shared_state)
|
|
146
155
|
return releases
|
|
147
156
|
|
|
148
|
-
soup = BeautifulSoup(r.content,
|
|
157
|
+
soup = BeautifulSoup(r.content, "html.parser")
|
|
149
158
|
|
|
150
159
|
# 1) New “Releases”
|
|
151
160
|
release_rows = soup.select("#releases_updates_list table tbody tr")
|
|
@@ -205,34 +214,42 @@ def al_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
205
214
|
mt_blocks = tr.find_all("div", class_="mt10")
|
|
206
215
|
for block in mt_blocks:
|
|
207
216
|
release_id = get_release_id(block)
|
|
208
|
-
release_info = parse_info_from_feed_entry(
|
|
217
|
+
release_info = parse_info_from_feed_entry(
|
|
218
|
+
block, raw_base_title, release_type
|
|
219
|
+
)
|
|
209
220
|
final_title = guess_title(shared_state, raw_base_title, release_info)
|
|
210
221
|
|
|
211
222
|
# Build payload using final_title
|
|
212
223
|
mb = 0 # size not available in feed
|
|
213
|
-
raw = f"{final_title}|{url}|{mirror}|{mb}|{release_id}||{hostname}".encode(
|
|
224
|
+
raw = f"{final_title}|{url}|{mirror}|{mb}|{release_id}||{hostname}".encode(
|
|
225
|
+
"utf-8"
|
|
226
|
+
)
|
|
214
227
|
payload = urlsafe_b64encode(raw).decode("utf-8")
|
|
215
228
|
link = f"{shared_state.values['internal_address']}/download/?payload={payload}"
|
|
216
229
|
|
|
217
230
|
# Append only unique releases
|
|
218
231
|
if final_title not in [r["details"]["title"] for r in releases]:
|
|
219
|
-
releases.append(
|
|
220
|
-
|
|
221
|
-
"
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
+
releases.append(
|
|
233
|
+
{
|
|
234
|
+
"details": {
|
|
235
|
+
"title": final_title,
|
|
236
|
+
"hostname": hostname,
|
|
237
|
+
"imdb_id": None,
|
|
238
|
+
"link": link,
|
|
239
|
+
"mirror": mirror,
|
|
240
|
+
"size": mb * 1024 * 1024,
|
|
241
|
+
"date": date_converted,
|
|
242
|
+
"source": url,
|
|
243
|
+
},
|
|
244
|
+
"type": "protected",
|
|
245
|
+
}
|
|
246
|
+
)
|
|
232
247
|
|
|
233
248
|
except Exception as e:
|
|
234
249
|
info(f"{hostname}: error parsing feed item: {e}")
|
|
235
|
-
mark_hostname_issue(
|
|
250
|
+
mark_hostname_issue(
|
|
251
|
+
hostname, "feed", str(e) if "e" in dir() else "Error occurred"
|
|
252
|
+
)
|
|
236
253
|
|
|
237
254
|
elapsed = time.time() - start_time
|
|
238
255
|
debug(f"Time taken: {elapsed:.2f}s ({hostname})")
|
|
@@ -243,19 +260,28 @@ def al_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
243
260
|
|
|
244
261
|
|
|
245
262
|
def extract_season(title: str) -> int | None:
|
|
246
|
-
match = re.search(r
|
|
263
|
+
match = re.search(r"(?i)(?:^|[^a-zA-Z0-9])S(\d{1,4})(?!\d)", title)
|
|
247
264
|
if match:
|
|
248
265
|
return int(match.group(1))
|
|
249
266
|
return None
|
|
250
267
|
|
|
251
268
|
|
|
252
|
-
def al_search(
|
|
253
|
-
|
|
269
|
+
def al_search(
|
|
270
|
+
shared_state,
|
|
271
|
+
start_time,
|
|
272
|
+
request_from,
|
|
273
|
+
search_string,
|
|
274
|
+
mirror=None,
|
|
275
|
+
season=None,
|
|
276
|
+
episode=None,
|
|
277
|
+
):
|
|
254
278
|
releases = []
|
|
255
279
|
host = shared_state.values["config"]("Hostnames").get(hostname)
|
|
256
280
|
|
|
257
281
|
if not "arr" in request_from.lower():
|
|
258
|
-
debug(
|
|
282
|
+
debug(
|
|
283
|
+
f'Skipping {request_from} search on "{hostname.upper()}" (unsupported media type)!'
|
|
284
|
+
)
|
|
259
285
|
return releases
|
|
260
286
|
|
|
261
287
|
if "Radarr" in request_from:
|
|
@@ -269,7 +295,7 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
269
295
|
|
|
270
296
|
imdb_id = shared_state.is_imdb_id(search_string)
|
|
271
297
|
if imdb_id:
|
|
272
|
-
title = get_localized_title(shared_state, imdb_id,
|
|
298
|
+
title = get_localized_title(shared_state, imdb_id, "de")
|
|
273
299
|
if not title:
|
|
274
300
|
info(f"{hostname}: no title for IMDb {imdb_id}")
|
|
275
301
|
return releases
|
|
@@ -280,21 +306,29 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
280
306
|
encoded_search_string = quote_plus(search_string)
|
|
281
307
|
|
|
282
308
|
try:
|
|
283
|
-
url = f
|
|
284
|
-
r = fetch_via_requests_session(
|
|
309
|
+
url = f"https://www.{host}/search?q={encoded_search_string}"
|
|
310
|
+
r = fetch_via_requests_session(
|
|
311
|
+
shared_state, method="GET", target_url=url, timeout=10
|
|
312
|
+
)
|
|
285
313
|
r.raise_for_status()
|
|
286
314
|
except Exception as e:
|
|
287
315
|
info(f"{hostname}: search load error: {e}")
|
|
288
|
-
mark_hostname_issue(
|
|
316
|
+
mark_hostname_issue(
|
|
317
|
+
hostname, "search", str(e) if "e" in dir() else "Error occurred"
|
|
318
|
+
)
|
|
289
319
|
invalidate_session(shared_state)
|
|
290
320
|
return releases
|
|
291
321
|
|
|
292
322
|
if r.history:
|
|
293
323
|
# If just one valid search result exists, AL skips the search result page
|
|
294
324
|
last_redirect = r.history[-1]
|
|
295
|
-
redirect_location = last_redirect.headers[
|
|
296
|
-
absolute_redirect_url = urljoin(
|
|
297
|
-
|
|
325
|
+
redirect_location = last_redirect.headers["Location"]
|
|
326
|
+
absolute_redirect_url = urljoin(
|
|
327
|
+
last_redirect.url, redirect_location
|
|
328
|
+
) # in case of relative URL
|
|
329
|
+
debug(
|
|
330
|
+
f"{search_string} redirected to {absolute_redirect_url} instead of search results page"
|
|
331
|
+
)
|
|
298
332
|
|
|
299
333
|
try:
|
|
300
334
|
soup = BeautifulSoup(r.text, "html.parser")
|
|
@@ -304,18 +338,18 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
304
338
|
|
|
305
339
|
results = [{"url": absolute_redirect_url, "title": page_title}]
|
|
306
340
|
else:
|
|
307
|
-
soup = BeautifulSoup(r.text,
|
|
341
|
+
soup = BeautifulSoup(r.text, "html.parser")
|
|
308
342
|
results = []
|
|
309
343
|
|
|
310
|
-
for panel in soup.select(
|
|
311
|
-
body = panel.find(
|
|
344
|
+
for panel in soup.select("div.panel.panel-default"):
|
|
345
|
+
body = panel.find("div", class_="panel-body")
|
|
312
346
|
if not body:
|
|
313
347
|
continue
|
|
314
348
|
|
|
315
|
-
title_tag = body.select_one(
|
|
349
|
+
title_tag = body.select_one("h4.title-list a[href]")
|
|
316
350
|
if not title_tag:
|
|
317
351
|
continue
|
|
318
|
-
url = title_tag[
|
|
352
|
+
url = title_tag["href"].strip()
|
|
319
353
|
name = title_tag.get_text(strip=True)
|
|
320
354
|
|
|
321
355
|
sanitized_search_string = shared_state.sanitize_string(search_string)
|
|
@@ -326,13 +360,13 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
326
360
|
debug(f"Matched search string '{search_string}' with result '{name}'")
|
|
327
361
|
|
|
328
362
|
type_label = None
|
|
329
|
-
for lbl in body.select(
|
|
330
|
-
href = lbl[
|
|
331
|
-
if
|
|
332
|
-
type_label =
|
|
363
|
+
for lbl in body.select("div.label-group a[href]"):
|
|
364
|
+
href = lbl["href"]
|
|
365
|
+
if "/anime-series" in href:
|
|
366
|
+
type_label = "series"
|
|
333
367
|
break
|
|
334
|
-
if
|
|
335
|
-
type_label =
|
|
368
|
+
if "/anime-movies" in href:
|
|
369
|
+
type_label = "movie"
|
|
336
370
|
break
|
|
337
371
|
|
|
338
372
|
if not type_label or type_label != valid_type:
|
|
@@ -347,7 +381,9 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
347
381
|
|
|
348
382
|
context = "recents_al"
|
|
349
383
|
threshold = 60
|
|
350
|
-
recently_searched = shared_state.get_recently_searched(
|
|
384
|
+
recently_searched = shared_state.get_recently_searched(
|
|
385
|
+
shared_state, context, threshold
|
|
386
|
+
)
|
|
351
387
|
entry = recently_searched.get(url, {})
|
|
352
388
|
ts = entry.get("timestamp")
|
|
353
389
|
use_cache = ts and ts > datetime.now() - timedelta(seconds=threshold)
|
|
@@ -357,7 +393,9 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
357
393
|
data_html = entry["html"]
|
|
358
394
|
else:
|
|
359
395
|
entry = {"timestamp": datetime.now()}
|
|
360
|
-
data_html = fetch_via_requests_session(
|
|
396
|
+
data_html = fetch_via_requests_session(
|
|
397
|
+
shared_state, method="GET", target_url=url, timeout=10
|
|
398
|
+
).text
|
|
361
399
|
|
|
362
400
|
entry["html"] = data_html
|
|
363
401
|
recently_searched[url] = entry
|
|
@@ -371,8 +409,13 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
371
409
|
for tab in download_tabs:
|
|
372
410
|
release_id += 1
|
|
373
411
|
|
|
374
|
-
release_info = parse_info_from_download_item(
|
|
375
|
-
|
|
412
|
+
release_info = parse_info_from_download_item(
|
|
413
|
+
tab,
|
|
414
|
+
content,
|
|
415
|
+
page_title=title,
|
|
416
|
+
release_type=valid_type,
|
|
417
|
+
requested_episode=episode,
|
|
418
|
+
)
|
|
376
419
|
|
|
377
420
|
# Parse date
|
|
378
421
|
date_td = tab.select_one("tr:has(th>i.fa-calendar-alt) td.modified")
|
|
@@ -384,15 +427,16 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
384
427
|
except Exception:
|
|
385
428
|
date_str = ""
|
|
386
429
|
else:
|
|
387
|
-
date_str = (datetime.utcnow() - timedelta(hours=1))
|
|
388
|
-
|
|
430
|
+
date_str = (datetime.utcnow() - timedelta(hours=1)).strftime(
|
|
431
|
+
"%a, %d %b %Y %H:%M:%S +0000"
|
|
432
|
+
)
|
|
389
433
|
|
|
390
434
|
# Parse filesize from the <tr> with <i class="fa-hdd">
|
|
391
435
|
size_td = tab.select_one("tr:has(th>i.fa-hdd) td")
|
|
392
436
|
mb = 0
|
|
393
437
|
if size_td:
|
|
394
438
|
size_text = size_td.get_text(strip=True)
|
|
395
|
-
candidates = re.findall(r
|
|
439
|
+
candidates = re.findall(r"(\d+(\.\d+)?\s*[A-Za-z]+)", size_text)
|
|
396
440
|
if candidates:
|
|
397
441
|
size_string = candidates[-1][0]
|
|
398
442
|
try:
|
|
@@ -422,32 +466,39 @@ def al_search(shared_state, start_time, request_from, search_string,
|
|
|
422
466
|
release_title = guess_title(shared_state, title, release_info)
|
|
423
467
|
|
|
424
468
|
if season and release_info.season != int(season):
|
|
425
|
-
debug(
|
|
469
|
+
debug(
|
|
470
|
+
f"Excluding {release_title} due to season mismatch: {release_info.season} != {season}"
|
|
471
|
+
)
|
|
426
472
|
continue
|
|
427
473
|
|
|
428
474
|
payload = urlsafe_b64encode(
|
|
429
|
-
f"{release_title}|{url}|{mirror}|{mb}|{release_id}|{imdb_id or ''}"
|
|
430
|
-
|
|
475
|
+
f"{release_title}|{url}|{mirror}|{mb}|{release_id}|{imdb_id or ''}".encode(
|
|
476
|
+
"utf-8"
|
|
477
|
+
)
|
|
431
478
|
).decode("utf-8")
|
|
432
479
|
link = f"{shared_state.values['internal_address']}/download/?payload={payload}"
|
|
433
480
|
|
|
434
|
-
releases.append(
|
|
435
|
-
|
|
436
|
-
"
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
481
|
+
releases.append(
|
|
482
|
+
{
|
|
483
|
+
"details": {
|
|
484
|
+
"title": release_title,
|
|
485
|
+
"hostname": hostname,
|
|
486
|
+
"imdb_id": imdb_id,
|
|
487
|
+
"link": link,
|
|
488
|
+
"mirror": mirror,
|
|
489
|
+
"size": mb * 1024 * 1024,
|
|
490
|
+
"date": date_str,
|
|
491
|
+
"source": f"{url}#download_{release_id}",
|
|
492
|
+
},
|
|
493
|
+
"type": "protected",
|
|
494
|
+
}
|
|
495
|
+
)
|
|
447
496
|
|
|
448
497
|
except Exception as e:
|
|
449
498
|
info(f"{hostname}: error parsing search item: {e}")
|
|
450
|
-
mark_hostname_issue(
|
|
499
|
+
mark_hostname_issue(
|
|
500
|
+
hostname, "search", str(e) if "e" in dir() else "Error occurred"
|
|
501
|
+
)
|
|
451
502
|
|
|
452
503
|
elapsed = time.time() - start_time
|
|
453
504
|
debug(f"Time taken: {elapsed:.2f}s ({hostname})")
|
quasarr/search/sources/by.py
CHANGED
|
@@ -12,9 +12,9 @@ from urllib.parse import quote_plus
|
|
|
12
12
|
import requests
|
|
13
13
|
from bs4 import BeautifulSoup
|
|
14
14
|
|
|
15
|
-
from quasarr.providers.hostname_issues import
|
|
15
|
+
from quasarr.providers.hostname_issues import clear_hostname_issue, mark_hostname_issue
|
|
16
16
|
from quasarr.providers.imdb_metadata import get_localized_title
|
|
17
|
-
from quasarr.providers.log import
|
|
17
|
+
from quasarr.providers.log import debug, info
|
|
18
18
|
|
|
19
19
|
hostname = "by"
|
|
20
20
|
supported_mirrors = ["rapidgator", "ddownload", "nitroflare"]
|
|
@@ -37,87 +37,108 @@ def extract_size(text):
|
|
|
37
37
|
m = re.match(r"(\d+(?:[.,]\d+)?)\s*([A-Za-z]+)", text)
|
|
38
38
|
if not m:
|
|
39
39
|
raise ValueError(f"Invalid size format: {text!r}")
|
|
40
|
-
size_str = m.group(1).replace(
|
|
40
|
+
size_str = m.group(1).replace(",", ".")
|
|
41
41
|
sizeunit = m.group(2)
|
|
42
42
|
size_float = float(size_str) # convert to float here
|
|
43
43
|
return {"size": size_float, "sizeunit": sizeunit}
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def _parse_posts(
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
def _parse_posts(
|
|
47
|
+
soup,
|
|
48
|
+
shared_state,
|
|
49
|
+
base_url,
|
|
50
|
+
password,
|
|
51
|
+
mirror_filter,
|
|
52
|
+
is_search=False,
|
|
53
|
+
request_from=None,
|
|
54
|
+
search_string=None,
|
|
55
|
+
season=None,
|
|
56
|
+
episode=None,
|
|
57
|
+
):
|
|
49
58
|
releases = []
|
|
50
59
|
if not is_search:
|
|
51
|
-
feed_container = soup.find(
|
|
60
|
+
feed_container = soup.find(
|
|
61
|
+
"table", class_="AUDIO_ITEMLIST"
|
|
62
|
+
) # it is actually called this way
|
|
52
63
|
candidates = []
|
|
53
64
|
if feed_container:
|
|
54
|
-
for tbl in feed_container.find_all(
|
|
65
|
+
for tbl in feed_container.find_all("table"):
|
|
55
66
|
if tbl.find(string=re.compile(r"Erstellt am:")):
|
|
56
67
|
candidates.append(tbl)
|
|
57
68
|
items = candidates
|
|
58
69
|
else:
|
|
59
|
-
search_table = soup.find(
|
|
70
|
+
search_table = soup.find("table", class_="SEARCH_ITEMLIST")
|
|
60
71
|
items = []
|
|
61
72
|
if search_table:
|
|
62
73
|
items = [
|
|
63
|
-
tr
|
|
64
|
-
|
|
74
|
+
tr
|
|
75
|
+
for tr in search_table.find_all("tr")
|
|
76
|
+
if tr.find("p", class_="TITLE")
|
|
77
|
+
and tr.find("p", class_="TITLE").find("a", href=True)
|
|
65
78
|
]
|
|
66
79
|
|
|
67
80
|
for entry in items:
|
|
68
|
-
if entry.find(
|
|
81
|
+
if entry.find("table"):
|
|
69
82
|
continue # Skip header rows
|
|
70
83
|
try:
|
|
71
84
|
if not is_search:
|
|
72
85
|
table = entry
|
|
73
86
|
# title & source
|
|
74
87
|
try:
|
|
75
|
-
link_tag = table.find(
|
|
88
|
+
link_tag = table.find("th").find("a")
|
|
76
89
|
except AttributeError:
|
|
77
|
-
link_tag = table.find(
|
|
90
|
+
link_tag = table.find("a")
|
|
78
91
|
title = link_tag.get_text(strip=True)
|
|
79
|
-
if
|
|
92
|
+
if "lazylibrarian" in request_from.lower():
|
|
80
93
|
# lazylibrarian can only detect specific date formats / issue numbering for magazines
|
|
81
94
|
title = shared_state.normalize_magazine_title(title)
|
|
82
95
|
else:
|
|
83
96
|
title = title.replace(" ", ".")
|
|
84
97
|
|
|
85
|
-
source = base_url + link_tag[
|
|
98
|
+
source = base_url + link_tag["href"]
|
|
86
99
|
# extract date and size
|
|
87
100
|
date_str = size_str = None
|
|
88
|
-
for row in table.find_all(
|
|
89
|
-
cols = row.find_all(
|
|
101
|
+
for row in table.find_all("tr", height=True):
|
|
102
|
+
cols = row.find_all("td")
|
|
90
103
|
if len(cols) == 2:
|
|
91
104
|
label = cols[0].get_text(strip=True)
|
|
92
105
|
val = cols[1].get_text(strip=True)
|
|
93
|
-
if label.startswith(
|
|
106
|
+
if label.startswith("Erstellt am"):
|
|
94
107
|
date_str = val
|
|
95
|
-
elif label.startswith(
|
|
108
|
+
elif label.startswith("Größe"):
|
|
96
109
|
size_str = val
|
|
97
|
-
published = convert_to_rss_date(date_str) if date_str else
|
|
98
|
-
size_info =
|
|
99
|
-
|
|
110
|
+
published = convert_to_rss_date(date_str) if date_str else ""
|
|
111
|
+
size_info = (
|
|
112
|
+
extract_size(size_str)
|
|
113
|
+
if size_str
|
|
114
|
+
else {"size": "0", "sizeunit": "MB"}
|
|
115
|
+
)
|
|
116
|
+
mb = float(size_info["size"])
|
|
100
117
|
size_bytes = int(mb * 1024 * 1024)
|
|
101
118
|
imdb_id = None
|
|
102
119
|
else:
|
|
103
120
|
row = entry
|
|
104
|
-
title_tag = row.find(
|
|
121
|
+
title_tag = row.find("p", class_="TITLE").find("a")
|
|
105
122
|
title = title_tag.get_text(strip=True)
|
|
106
|
-
if
|
|
123
|
+
if "lazylibrarian" in request_from.lower():
|
|
107
124
|
# lazylibrarian can only detect specific date formats / issue numbering for magazines
|
|
108
125
|
title = shared_state.normalize_magazine_title(title)
|
|
109
126
|
else:
|
|
110
127
|
title = title.replace(" ", ".")
|
|
111
|
-
if not (
|
|
128
|
+
if not (
|
|
129
|
+
RESOLUTION_REGEX.search(title) or CODEC_REGEX.search(title)
|
|
130
|
+
):
|
|
112
131
|
continue
|
|
113
132
|
|
|
114
|
-
if not shared_state.is_valid_release(
|
|
133
|
+
if not shared_state.is_valid_release(
|
|
134
|
+
title, request_from, search_string, season, episode
|
|
135
|
+
):
|
|
115
136
|
continue
|
|
116
|
-
if XXX_REGEX.search(title) and
|
|
137
|
+
if XXX_REGEX.search(title) and "xxx" not in search_string.lower():
|
|
117
138
|
continue
|
|
118
139
|
|
|
119
|
-
source = base_url + title_tag[
|
|
120
|
-
date_cell = row.find_all(
|
|
140
|
+
source = base_url + title_tag["href"]
|
|
141
|
+
date_cell = row.find_all("td")[2]
|
|
121
142
|
date_str = date_cell.get_text(strip=True)
|
|
122
143
|
published = convert_to_rss_date(date_str)
|
|
123
144
|
size_bytes = 0
|
|
@@ -127,21 +148,25 @@ def _parse_posts(soup, shared_state, base_url, password, mirror_filter,
|
|
|
127
148
|
payload = urlsafe_b64encode(
|
|
128
149
|
f"{title}|{source}|{mirror_filter}|{mb}|{password}|{imdb_id}|{hostname}".encode()
|
|
129
150
|
).decode()
|
|
130
|
-
link =
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
151
|
+
link = (
|
|
152
|
+
f"{shared_state.values['internal_address']}/download/?payload={payload}"
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
releases.append(
|
|
156
|
+
{
|
|
157
|
+
"details": {
|
|
158
|
+
"title": title,
|
|
159
|
+
"hostname": hostname,
|
|
160
|
+
"imdb_id": imdb_id,
|
|
161
|
+
"link": link,
|
|
162
|
+
"mirror": mirror_filter,
|
|
163
|
+
"size": size_bytes,
|
|
164
|
+
"date": published,
|
|
165
|
+
"source": source,
|
|
166
|
+
},
|
|
167
|
+
"type": "protected",
|
|
168
|
+
}
|
|
169
|
+
)
|
|
145
170
|
except Exception as e:
|
|
146
171
|
debug(f"Error parsing {hostname.upper()}: {e}")
|
|
147
172
|
continue
|
|
@@ -150,7 +175,7 @@ def _parse_posts(soup, shared_state, base_url, password, mirror_filter,
|
|
|
150
175
|
|
|
151
176
|
|
|
152
177
|
def by_feed(shared_state, start_time, request_from, mirror=None):
|
|
153
|
-
by = shared_state.values[
|
|
178
|
+
by = shared_state.values["config"]("Hostnames").get(hostname)
|
|
154
179
|
password = by
|
|
155
180
|
|
|
156
181
|
if "lazylibrarian" in request_from.lower():
|
|
@@ -162,15 +187,24 @@ def by_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
162
187
|
|
|
163
188
|
base_url = f"https://{by}"
|
|
164
189
|
url = f"{base_url}/{feed_type}"
|
|
165
|
-
headers = {
|
|
190
|
+
headers = {"User-Agent": shared_state.values["user_agent"]}
|
|
166
191
|
try:
|
|
167
192
|
r = requests.get(url, headers=headers, timeout=30)
|
|
168
193
|
r.raise_for_status()
|
|
169
|
-
soup = BeautifulSoup(r.content,
|
|
170
|
-
releases = _parse_posts(
|
|
194
|
+
soup = BeautifulSoup(r.content, "html.parser")
|
|
195
|
+
releases = _parse_posts(
|
|
196
|
+
soup,
|
|
197
|
+
shared_state,
|
|
198
|
+
base_url,
|
|
199
|
+
password,
|
|
200
|
+
request_from=request_from,
|
|
201
|
+
mirror_filter=mirror,
|
|
202
|
+
)
|
|
171
203
|
except Exception as e:
|
|
172
204
|
info(f"Error loading {hostname.upper()} feed: {e}")
|
|
173
|
-
mark_hostname_issue(
|
|
205
|
+
mark_hostname_issue(
|
|
206
|
+
hostname, "feed", str(e) if "e" in dir() else "Error occurred"
|
|
207
|
+
)
|
|
174
208
|
releases = []
|
|
175
209
|
debug(f"Time taken: {time.time() - start_time:.2f}s ({hostname})")
|
|
176
210
|
|
|
@@ -179,13 +213,21 @@ def by_feed(shared_state, start_time, request_from, mirror=None):
|
|
|
179
213
|
return releases
|
|
180
214
|
|
|
181
215
|
|
|
182
|
-
def by_search(
|
|
183
|
-
|
|
216
|
+
def by_search(
|
|
217
|
+
shared_state,
|
|
218
|
+
start_time,
|
|
219
|
+
request_from,
|
|
220
|
+
search_string,
|
|
221
|
+
mirror=None,
|
|
222
|
+
season=None,
|
|
223
|
+
episode=None,
|
|
224
|
+
):
|
|
225
|
+
by = shared_state.values["config"]("Hostnames").get(hostname)
|
|
184
226
|
password = by
|
|
185
227
|
|
|
186
228
|
imdb_id = shared_state.is_imdb_id(search_string)
|
|
187
229
|
if imdb_id:
|
|
188
|
-
title = get_localized_title(shared_state, imdb_id,
|
|
230
|
+
title = get_localized_title(shared_state, imdb_id, "de")
|
|
189
231
|
if not title:
|
|
190
232
|
info(f"Could not extract title from IMDb-ID {imdb_id}")
|
|
191
233
|
return []
|
|
@@ -194,19 +236,28 @@ def by_search(shared_state, start_time, request_from, search_string, mirror=None
|
|
|
194
236
|
base_url = f"https://{by}"
|
|
195
237
|
q = quote_plus(search_string)
|
|
196
238
|
url = f"{base_url}/?q={q}"
|
|
197
|
-
headers = {
|
|
239
|
+
headers = {"User-Agent": shared_state.values["user_agent"]}
|
|
198
240
|
try:
|
|
199
241
|
r = requests.get(url, headers=headers, timeout=10)
|
|
200
242
|
r.raise_for_status()
|
|
201
|
-
soup = BeautifulSoup(r.content,
|
|
243
|
+
soup = BeautifulSoup(r.content, "html.parser")
|
|
202
244
|
releases = _parse_posts(
|
|
203
|
-
soup,
|
|
204
|
-
|
|
205
|
-
|
|
245
|
+
soup,
|
|
246
|
+
shared_state,
|
|
247
|
+
base_url,
|
|
248
|
+
password,
|
|
249
|
+
mirror_filter=mirror,
|
|
250
|
+
is_search=True,
|
|
251
|
+
request_from=request_from,
|
|
252
|
+
search_string=search_string,
|
|
253
|
+
season=season,
|
|
254
|
+
episode=episode,
|
|
206
255
|
)
|
|
207
256
|
except Exception as e:
|
|
208
257
|
info(f"Error loading {hostname.upper()} search: {e}")
|
|
209
|
-
mark_hostname_issue(
|
|
258
|
+
mark_hostname_issue(
|
|
259
|
+
hostname, "search", str(e) if "e" in dir() else "Error occurred"
|
|
260
|
+
)
|
|
210
261
|
releases = []
|
|
211
262
|
debug(f"Time taken: {time.time() - start_time:.2f}s ({hostname})")
|
|
212
263
|
|