quasarr 1.20.8__py3-none-any.whl → 1.21.0__py3-none-any.whl

quasarr/providers/sessions/dl.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+# Quasarr
+# Project by https://github.com/rix1337
+
+import base64
+import pickle
+
+import requests
+from bs4 import BeautifulSoup
+
+from quasarr.providers.log import info, debug
+
+hostname = "dl"
+
+
+def create_and_persist_session(shared_state):
+    """
+    Create and persist a session using user and password.
+    
+    Args:
+        shared_state: Shared state object
+    
+    Returns:
+        requests.Session or None
+    """
+    cfg = shared_state.values["config"]("Hostnames")
+    host = cfg.get(hostname)
+    credentials_cfg = shared_state.values["config"](hostname.upper())
+    
+    user = credentials_cfg.get("user")
+    password = credentials_cfg.get("password")
+
+    if not user or not password:
+        info(f'Missing credentials for: "{hostname}" - user and password are required')
+        return None
+
+    sess = requests.Session()
+    
+    # Set user agent
+    ua = shared_state.values["user_agent"]
+    sess.headers.update({'User-Agent': ua})
+    
+    try:
+        # Step 1: Get login page to retrieve CSRF token
+        login_page_url = f'https://www.{host}/login/'
+        login_page = sess.get(login_page_url, timeout=30)
+        
+        if login_page.status_code != 200:
+            info(f'Failed to load login page for: "{hostname}" - Status {login_page.status_code}')
+            return None
+        
+        # Extract CSRF token from login form
+        soup = BeautifulSoup(login_page.text, 'html.parser')
+        csrf_input = soup.find('input', {'name': '_xfToken'})
+        
+        if not csrf_input or not csrf_input.get('value'):
+            info(f'Could not find CSRF token on login page for: "{hostname}"')
+            return None
+        
+        csrf_token = csrf_input['value']
+        
+        # Step 2: Submit login form
+        login_data = {
+            'login': user,
+            'password': password,
+            '_xfToken': csrf_token,
+            'remember': '1',
+            '_xfRedirect': f'https://www.{host}/'
+        }
+        
+        login_url = f'https://www.{host}/login/login'
+        login_response = sess.post(login_url, data=login_data, timeout=30)
+        
+        # Step 3: Verify login success
+        # Check if we're logged in by accessing the main page
+        verify_response = sess.get(f'https://www.{host}/', timeout=30)
+        
+        if 'data-logged-in="true"' not in verify_response.text:
+            info(f'Login verification failed for: "{hostname}" - invalid credentials or login failed')
+            return None
+        
+        info(f'Session successfully created for: "{hostname}" using user/password')
+    except Exception as e:
+        info(f'Failed to create session for: "{hostname}" - {e}')
+        return None
+
+    # Persist session to database
+    blob = pickle.dumps(sess)
+    token = base64.b64encode(blob).decode("utf-8")
+    shared_state.values["database"]("sessions").update_store(hostname, token)
+    
+    return sess
+
+
+def retrieve_and_validate_session(shared_state):
+    """
+    Retrieve session from database or create a new one.
+    
+    Args:
+        shared_state: Shared state object
+    
+    Returns:
+        requests.Session or None
+    """
+    db = shared_state.values["database"]("sessions")
+    token = db.retrieve(hostname)
+    if not token:
+        return create_and_persist_session(shared_state)
+
+    try:
+        blob = base64.b64decode(token.encode("utf-8"))
+        sess = pickle.loads(blob)
+        if not isinstance(sess, requests.Session):
+            raise ValueError("Not a Session")
+    except Exception as e:
+        debug(f"{hostname}: session load failed: {e}")
+        return create_and_persist_session(shared_state)
+
+    return sess
+
+
+def invalidate_session(shared_state):
+    """
+    Invalidate the current session.
+    
+    Args:
+        shared_state: Shared state object
+    """
+    db = shared_state.values["database"]("sessions")
+    db.delete(hostname)
+    debug(f'Session for "{hostname}" marked as invalid!')
+
+
+def _persist_session_to_db(shared_state, sess):
+    """
+    Serialize & store the given requests.Session into the database under `hostname`.
+    
+    Args:
+        shared_state: Shared state object
+        sess: requests.Session to persist
+    """
+    blob = pickle.dumps(sess)
+    token = base64.b64encode(blob).decode("utf-8")
+    shared_state.values["database"]("sessions").update_store(hostname, token)
+
+
+def fetch_via_requests_session(shared_state, method: str, target_url: str, post_data: dict = None, get_params: dict = None, timeout: int = 30):
+    """
+    Execute request using the session.
+    
+    Args:
+        shared_state: Shared state object
+        method: "GET" or "POST"
+        target_url: URL to fetch
+        post_data: POST data (for POST requests)
+        get_params: URL parameters (for GET requests)
+        timeout: Request timeout in seconds
+    
+    Returns:
+        Response object
+    """
+    sess = retrieve_and_validate_session(shared_state)
+    if not sess:
+        raise Exception(f"Could not retrieve valid session for {hostname}")
+
+    # Execute request
+    if method.upper() == "GET":
+        resp = sess.get(target_url, params=get_params, timeout=timeout)
+    else:  # POST
+        resp = sess.post(target_url, data=post_data, timeout=timeout)
+
+    # Re-persist cookies, since the site might have modified them during the request
+    _persist_session_to_db(shared_state, sess)
+
+    return resp

quasarr/providers/shared_state.py

@@ -188,6 +188,7 @@ def connect_device():
 
 def get_device():
     attempts = 0
+    last_backoff_change = 0  # Track when we last changed backoff strategy
 
     while True:
         try:
@@ -199,14 +200,30 @@ def get_device():
 
         update("device", False)
 
-        if attempts % 10 == 0:
-            info(
-                f"WARNING: {attempts} consecutive JDownloader connection errors. Please check your credentials!")
-        time.sleep(3)
+        # Determine sleep time based on failure count
+        if attempts <= 10:
+            # First 10 failures: 3 seconds
+            sleep_time = 3
+            if attempts == 10:
+                info(f"WARNING: {attempts} consecutive JDownloader connection errors. Switching to 1-minute intervals.")
+        elif attempts <= 15:
+            # Next 5 failures (11-15): 1 minute
+            sleep_time = 60
+            if attempts % 10 == 0:
+                info(f"WARNING: {attempts} consecutive JDownloader connection errors. Please check your credentials!")
+            if attempts == 15:
+                info(f"WARNING: Still failing after {attempts} attempts. Switching to 5-minute intervals.")
+        else:
+            # After 15 failures: 5 minutes
+            sleep_time = 300
+            if attempts % 10 == 0:
+                info(f"WARNING: {attempts} consecutive JDownloader connection errors. Please check your credentials!")
 
         if connect_device():
             break
 
+        time.sleep(sleep_time)
+
     return values["device"]
 
 
@@ -608,7 +625,6 @@ def is_valid_release(title: str,
                 debug(f"Skipping {title!r} as it doesn't match sanitized search string: {search_string!r}")
                 return False
 
-
         # if it's a movie search, don't allow any TV show titles (check for NO season or episode tags in the title)
         if is_movie_search:
             if not MOVIE_REGEX.match(title):

quasarr/providers/version.py

@@ -8,7 +8,7 @@ import requests
 
 
 def get_version():
-    return "1.20.8"
+    return "1.21.0"
 
 
 def get_latest_version():

quasarr/search/__init__.py

@@ -10,6 +10,7 @@ from quasarr.search.sources.al import al_feed, al_search
 from quasarr.search.sources.by import by_feed, by_search
 from quasarr.search.sources.dd import dd_search, dd_feed
 from quasarr.search.sources.dj import dj_search, dj_feed
+from quasarr.search.sources.dl import dl_search, dl_feed
 from quasarr.search.sources.dt import dt_feed, dt_search
 from quasarr.search.sources.dw import dw_feed, dw_search
 from quasarr.search.sources.fx import fx_feed, fx_search
@@ -21,6 +22,7 @@ from quasarr.search.sources.sf import sf_feed, sf_search
 from quasarr.search.sources.sj import sj_search, sj_feed
 from quasarr.search.sources.sl import sl_feed, sl_search
 from quasarr.search.sources.wd import wd_feed, wd_search
+from quasarr.search.sources.wx import wx_feed, wx_search
 
 
 def get_search_results(shared_state, request_from, imdb_id="", search_phrase="", mirror=None, season="", episode=""):
@@ -34,6 +36,7 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
     al = shared_state.values["config"]("Hostnames").get("al")
     by = shared_state.values["config"]("Hostnames").get("by")
     dd = shared_state.values["config"]("Hostnames").get("dd")
+    dl = shared_state.values["config"]("Hostnames").get("dl")
     dt = shared_state.values["config"]("Hostnames").get("dt")
     dj = shared_state.values["config"]("Hostnames").get("dj")
     dw = shared_state.values["config"]("Hostnames").get("dw")
@@ -46,6 +49,7 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
     sj = shared_state.values["config"]("Hostnames").get("sj")
     sl = shared_state.values["config"]("Hostnames").get("sl")
     wd = shared_state.values["config"]("Hostnames").get("wd")
+    wx = shared_state.values["config"]("Hostnames").get("wx")
 
     start_time = time.time()
 
@@ -56,6 +60,7 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
         (al, al_search),
         (by, by_search),
         (dd, dd_search),
+        (dl, dl_search),
         (dt, dt_search),
         (dj, dj_search),
         (dw, dw_search),
@@ -68,11 +73,13 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
         (sj, sj_search),
         (sl, sl_search),
         (wd, wd_search),
+        (wx, wx_search),
     ]
 
     # LazyLibrarian uses search_phrase for searches
     phrase_map = [
         (by, by_search),
+        (dl, dl_search),
         (dt, dt_search),
         (nx, nx_search),
         (sl, sl_search),
@@ -85,6 +92,7 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
         (by, by_feed),
         (dd, dd_feed),
         (dj, dj_feed),
+        (dl, dl_feed),
         (dt, dt_feed),
         (dw, dw_feed),
         (fx, fx_feed),
@@ -96,6 +104,7 @@ def get_search_results(shared_state, request_from, imdb_id="", search_phrase="",
         (sj, sj_feed),
         (sl, sl_feed),
         (wd, wd_feed),
+        (wx, wx_feed),
     ]
 
     if imdb_id:  # only Radarr/Sonarr are using imdb_id

quasarr/search/sources/dl.py

@@ -0,0 +1,316 @@
+# -*- coding: utf-8 -*-
+# Quasarr
+# Project by https://github.com/rix1337
+
+import re
+import time
+import warnings
+from base64 import urlsafe_b64encode
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from html import unescape
+
+from bs4 import BeautifulSoup
+from bs4 import XMLParsedAsHTMLWarning
+
+from quasarr.providers.imdb_metadata import get_localized_title
+from quasarr.providers.log import info, debug
+from quasarr.providers.sessions.dl import retrieve_and_validate_session, invalidate_session, fetch_via_requests_session
+
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)  # we dont want to use lxml
+
+hostname = "dl"
+supported_mirrors = []
+
+
+def normalize_title_for_sonarr(title):
+    """
+    Normalize title for Sonarr by replacing spaces with dots.
+    """
+    title = title.replace(' ', '.')
+    title = re.sub(r'\s*-\s*', '-', title)
+    title = re.sub(r'\.\-\.', '-', title)
+    title = re.sub(r'\.{2,}', '.', title)
+    title = title.strip('.')
+    return title
+
+
+def dl_feed(shared_state, start_time, request_from, mirror=None):
+    """
+    Parse the RSS feed and return releases.
+    """
+    releases = []
+    host = shared_state.values["config"]("Hostnames").get(hostname)
+
+    if not host:
+        debug(f"{hostname}: hostname not configured")
+        return releases
+
+    try:
+        sess = retrieve_and_validate_session(shared_state)
+        if not sess:
+            info(f"Could not retrieve valid session for {host}")
+            return releases
+
+        # Instead we should parse the HTML for the correct *arr client
+        rss_url = f'https://www.{host}/forums/-/index.rss'
+        response = sess.get(rss_url, timeout=30)
+
+        if response.status_code != 200:
+            info(f"{hostname}: RSS feed returned status {response.status_code}")
+            return releases
+
+        soup = BeautifulSoup(response.content, 'html.parser')
+        items = soup.find_all('item')
+
+        if not items:
+            debug(f"{hostname}: No entries found in RSS feed")
+            return releases
+
+        for item in items:
+            try:
+                title_tag = item.find('title')
+                if not title_tag:
+                    continue
+
+                title = title_tag.get_text(strip=True)
+                if not title:
+                    continue
+
+                title = unescape(title)
+                title = title.replace(']]>', '').replace('<![CDATA[', '')
+                title = normalize_title_for_sonarr(title)
+
+                item_text = item.get_text()
+                thread_url = None
+                match = re.search(r'https://[^\s]+/threads/[^\s]+', item_text)
+                if match:
+                    thread_url = match.group(0)
+                if not thread_url:
+                    continue
+
+                pub_date = item.find('pubdate')
+                if pub_date:
+                    date_str = pub_date.get_text(strip=True)
+                else:
+                    # Fallback: use current time if no pubDate found
+                    date_str = datetime.now().strftime("%a, %d %b %Y %H:%M:%S +0000")
+
+                mb = 0
+                imdb_id = None
+                password = ""
+
+                payload = urlsafe_b64encode(
+                    f"{title}|{thread_url}|{mirror}|{mb}|{password}|{imdb_id or ''}".encode("utf-8")
+                ).decode("utf-8")
+                link = f"{shared_state.values['internal_address']}/download/?payload={payload}"
+
+                releases.append({
+                    "details": {
+                        "title": title,
+                        "hostname": hostname,
+                        "imdb_id": imdb_id,
+                        "link": link,
+                        "mirror": mirror,
+                        "size": mb * 1024 * 1024,
+                        "date": date_str,
+                        "source": thread_url
+                    },
+                    "type": "protected"
+                })
+
+            except Exception as e:
+                debug(f"{hostname}: error parsing RSS entry: {e}")
+                continue
+
+    except Exception as e:
+        info(f"{hostname}: RSS feed error: {e}")
+        invalidate_session(shared_state)
+
+    elapsed = time.time() - start_time
+    debug(f"Time taken: {elapsed:.2f}s ({hostname})")
+    return releases
+
+
+def _search_single_page(shared_state, host, search_string, search_id, page_num, imdb_id, mirror, request_from, season,
+                        episode):
+    """
+    Search a single page. This function is called in parallel for each page.
+    """
+    page_releases = []
+
+    try:
+        if page_num == 1:
+            search_params = {
+                'keywords': search_string,
+                'c[title_only]': 1
+            }
+            search_url = f'https://www.{host}/search/search'
+        else:
+            if not search_id:
+                return page_releases, None
+
+            search_params = {
+                'page': page_num,
+                'q': search_string,
+                'o': 'relevance'
+            }
+            search_url = f'https://www.{host}/search/{search_id}/'
+
+        search_response = fetch_via_requests_session(shared_state, method="GET",
+                                                     target_url=search_url,
+                                                     get_params=search_params,
+                                                     timeout=10)
+
+        if search_response.status_code != 200:
+            debug(f"{hostname}: [Page {page_num}] returned status {search_response.status_code}")
+            return page_releases, None
+
+        # Extract search ID from first page
+        extracted_search_id = None
+        if page_num == 1:
+            match = re.search(r'/search/(\d+)/', search_response.url)
+            if match:
+                extracted_search_id = match.group(1)
+                debug(f"{hostname}: [Page 1] Extracted search ID: {extracted_search_id}")
+
+        soup = BeautifulSoup(search_response.text, 'html.parser')
+        result_items = soup.select('li.block-row')
+
+        if not result_items:
+            debug(f"{hostname}: [Page {page_num}] found 0 results")
+            return page_releases, extracted_search_id
+
+        debug(f"{hostname}: [Page {page_num}] found {len(result_items)} results")
+
+        for item in result_items:
+            try:
+                title_elem = item.select_one('h3.contentRow-title a')
+                if not title_elem:
+                    continue
+
+                title = title_elem.get_text(separator=' ', strip=True)
+                title = re.sub(r'\s+', ' ', title)
+                title = unescape(title)
+                title_normalized = normalize_title_for_sonarr(title)
+
+                thread_url = title_elem.get('href')
+                if thread_url.startswith('/'):
+                    thread_url = f"https://www.{host}{thread_url}"
+
+                if not shared_state.is_valid_release(title_normalized, request_from, search_string, season, episode):
+                    continue
+
+                minor_info = item.select_one('div.contentRow-minor')
+                date_str = ""
+                if minor_info:
+                    date_elem = minor_info.select_one('time.u-dt')
+                    if date_elem:
+                        date_str = date_elem.get('datetime', '')
+
+                # Fallback: use current time if no date found
+                if not date_str:
+                    date_str = datetime.now().strftime("%a, %d %b %Y %H:%M:%S +0000")
+
+                mb = 0
+                password = ""
+
+                payload = urlsafe_b64encode(
+                    f"{title_normalized}|{thread_url}|{mirror}|{mb}|{password}|{imdb_id or ''}".encode("utf-8")
+                ).decode("utf-8")
+                link = f"{shared_state.values['internal_address']}/download/?payload={payload}"
+
+                page_releases.append({
+                    "details": {
+                        "title": title_normalized,
+                        "hostname": hostname,
+                        "imdb_id": imdb_id,
+                        "link": link,
+                        "mirror": mirror,
+                        "size": mb * 1024 * 1024,
+                        "date": date_str,
+                        "source": thread_url
+                    },
+                    "type": "protected"
+                })
+
+            except Exception as e:
+                debug(f"{hostname}: [Page {page_num}] error parsing item: {e}")
+
+        return page_releases, extracted_search_id
+
+    except Exception as e:
+        info(f"{hostname}: [Page {page_num}] error: {e}")
+        return page_releases, None
+
+
+def dl_search(shared_state, start_time, request_from, search_string,
+              mirror=None, season=None, episode=None):
+    """
+    Search with parallel pagination (max 5 pages) to find best quality releases.
+    Requests are fired in parallel to minimize search time.
+    """
+    releases = []
+    host = shared_state.values["config"]("Hostnames").get(hostname)
+
+    imdb_id = shared_state.is_imdb_id(search_string)
+    if imdb_id:
+        title = get_localized_title(shared_state, imdb_id, 'de')
+        if not title:
+            info(f"{hostname}: no title for IMDb {imdb_id}")
+            return releases
+        search_string = title
+
+    search_string = unescape(search_string)
+    max_pages = 5
+
+    info(
+        f"{hostname}: Starting parallel paginated search for '{search_string}' (Season: {season}, Episode: {episode}) - up to {max_pages} pages")
+
+    try:
+        sess = retrieve_and_validate_session(shared_state)
+        if not sess:
+            info(f"Could not retrieve valid session for {host}")
+            return releases
+
+        # First, do page 1 to get the search ID
+        page_1_releases, search_id = _search_single_page(
+            shared_state, host, search_string, None, 1,
+            imdb_id, mirror, request_from, season, episode
+        )
+        releases.extend(page_1_releases)
+
+        if not search_id:
+            info(f"{hostname}: Could not extract search ID, stopping pagination")
+            return releases
+
+        # Now fire remaining pages in parallel
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            futures = {}
+            for page_num in range(2, max_pages + 1):
+                future = executor.submit(
+                    _search_single_page,
+                    shared_state, host, search_string, search_id, page_num,
+                    imdb_id, mirror, request_from, season, episode
+                )
+                futures[future] = page_num
+
+            for future in as_completed(futures):
+                page_num = futures[future]
+                try:
+                    page_releases, _ = future.result()
+                    releases.extend(page_releases)
+                    debug(f"{hostname}: [Page {page_num}] completed with {len(page_releases)} valid releases")
+                except Exception as e:
+                    info(f"{hostname}: [Page {page_num}] failed: {e}")
+
+    except Exception as e:
+        info(f"{hostname}: search error: {e}")
+        invalidate_session(shared_state)
+
+    info(f"{hostname}: FINAL - Found {len(releases)} valid releases - providing to {request_from}")
+
+    elapsed = time.time() - start_time
+    debug(f"Time taken: {elapsed:.2f}s ({hostname})")
+
+    return releases