quasarr 1.29.0__py3-none-any.whl → 1.31.0__py3-none-any.whl

quasarr/downloads/__init__.py

@@ -25,6 +25,7 @@ from quasarr.downloads.sources.wx import get_wx_download_links
 from quasarr.providers.log import info
 from quasarr.providers.notifications import send_discord_message
 from quasarr.providers.statistics import StatsHelper
+from quasarr.providers.utils import filter_offline_links
 
 # =============================================================================
 # CRYPTER CONFIGURATION
@@ -184,6 +185,12 @@ def process_links(shared_state, source_result, title, password, package_id, imdb
                     reason=f'All links are offline or IP is banned for "{title}" on {label} - "{source_url}"')
     links = valid_links
 
+    # Filter out verifiably offline links
+    links = filter_offline_links(links, shared_state=shared_state, log_func=info)
+    if not links:
+        return fail(title, package_id, shared_state,
+                    reason=f'All verifiable links are offline for "{title}" on {label} - "{source_url}"')
+
     classified = classify_links(links, shared_state)
 
     # PRIORITY 1: Direct hoster links

quasarr/downloads/linkcrypters/filecrypt.py

@@ -395,42 +395,8 @@ def get_filecrypt_links(shared_state, token, title, url, password=None, mirror=N
                     links.extend(DLC(shared_state, dlc_file).decrypt())
                 except:
                     debug("DLC fallback failed, trying button fallback.")
-                    info("DLC not found! Falling back to first available download Button...")
-
-                    base_url = urlparse(url).netloc
-                    phpsessid = session.cookies.get('PHPSESSID')
-                    if not phpsessid:
-                        info("PHPSESSID cookie not found! Cannot proceed with download links extraction.")
-                        debug("Missing PHPSESSID cookie.")
-                        return False
-
-                    results = []
-                    debug("Parsing fallback buttons for download links.")
-
-                    for button in soup.find_all('button'):
-                        data_attrs = [v for k, v in button.attrs.items() if k.startswith('data-') and k != 'data-i18n']
-                        if not data_attrs:
-                            continue
-
-                        link_id = data_attrs[0]
-                        row = button.find_parent('tr')
-                        mirror_tag = row.find('a', class_='external_link') if row else None
-                        mirror_name = mirror_tag.get_text(strip=True) if mirror_tag else 'unknown'
-                        full_url = f"http://{base_url}/Link/{link_id}.html"
-                        results.append((full_url, mirror_name))
-
-                    sorted_results = sorted(results, key=lambda x: 0 if 'rapidgator' in x[1].lower() else 1)
-                    debug(f"Found {len(sorted_results)} fallback link candidates.")
-
-                    for result_url, mirror in sorted_results:
-                        info("You must solve circlecaptcha separately!")
-                        debug(f'Session "{phpsessid}" for {result_url} will not live long. Submit new CAPTCHA quickly!')
-                        return {
-                            "status": "replaced",
-                            "replace_url": result_url,
-                            "mirror": mirror,
-                            "session": phpsessid
-                        }
+                    info("Click'n'Load and DLC not found. Please use the fallback userscript instead!")
+                    return False
 
     if not links:
         info("No links found in Filecrypt response!")

quasarr/downloads/packages/__init__.py

@@ -6,6 +6,7 @@ import json
 from collections import defaultdict
 from urllib.parse import urlparse
 
+from quasarr.providers.jd_cache import JDPackageCache
 from quasarr.providers.log import info, debug
 from quasarr.providers.myjd_api import TokenExpiredException, RequestTimeoutException, MYJDException
 
@@ -102,9 +103,23 @@ def format_eta(seconds):
         return f"{hours:02}:{minutes:02}:{seconds:02}"
 
 
-def get_packages(shared_state):
+def get_packages(shared_state, _cache=None):
+    """
+    Get all packages from protected DB, failed DB, linkgrabber, and downloader.
+
+    Args:
+        shared_state: The shared state object
+        _cache: INTERNAL USE ONLY. Used by delete_package() to share cached data
+                within a single request. External callers should never pass this.
+    """
     packages = []
 
+    # Create cache for this request - only valid for duration of this call
+    if _cache is None:
+        _cache = JDPackageCache(shared_state.get_device())
+
+    cache = _cache  # Use shorter name internally
+
     protected_packages = shared_state.get_db("protected").retrieve_all_titles()
     if protected_packages:
         for package in protected_packages:
@@ -152,16 +167,15 @@ def get_packages(shared_state):
                 "comment": package_id,
                 "uuid": package_id
             })
-    try:
-        linkgrabber_packages = shared_state.get_device().linkgrabber.query_packages()
-        linkgrabber_links = shared_state.get_device().linkgrabber.query_links()
-    except (TokenExpiredException, RequestTimeoutException, MYJDException):
-        linkgrabber_packages = []
-        linkgrabber_links = []
+
+    # Use cached queries instead of direct API calls
+    linkgrabber_packages = cache.linkgrabber_packages
+    linkgrabber_links = cache.linkgrabber_links
 
     if linkgrabber_packages:
         for package in linkgrabber_packages:
-            comment = get_links_comment(package, shared_state.get_device().linkgrabber.query_links())
+            # Use cached linkgrabber_links instead of re-querying
+            comment = get_links_comment(package, linkgrabber_links)
             link_details = get_links_status(package, linkgrabber_links, is_archive=False)
 
             error = link_details["error"]
@@ -184,25 +198,21 @@ def get_packages(shared_state):
                 "uuid": package.get("uuid"),
                 "error": error
             })
-    try:
-        downloader_packages = shared_state.get_device().downloads.query_packages()
-        downloader_links = shared_state.get_device().downloads.query_links()
-    except (TokenExpiredException, RequestTimeoutException, MYJDException):
-        downloader_packages = []
-        downloader_links = []
+
+    # Use cached queries instead of direct API calls
+    downloader_packages = cache.downloader_packages
+    downloader_links = cache.downloader_links
 
     if downloader_packages and downloader_links:
+        # Get all package UUIDs that contain archives (uses link data, fallback to single API call)
+        archive_package_uuids = cache.get_archive_package_uuids(downloader_packages, downloader_links)
+
         for package in downloader_packages:
             comment = get_links_comment(package, downloader_links)
 
-            # Check if package is actually archived/extracted using archive info
-            is_archive = False
-            try:
-                archive_info = shared_state.get_device().extraction.get_archive_info([], [package.get("uuid")])
-                is_archive = True if archive_info and archive_info[0] else False
-            except:
-                # On error, don't assume it's an archive - check bytes instead
-                pass
+            # Check if this package contains any archive files
+            package_uuid = package.get("uuid")
+            is_archive = package_uuid in archive_package_uuids if package_uuid else False
 
             link_details = get_links_status(package, downloader_links, is_archive)
 
@@ -250,7 +260,7 @@ def get_packages(shared_state):
             time_left = "23:59:59"
             if package["type"] == "linkgrabber":
                 details = package["details"]
-                name = f"[Linkgrabber] {details["name"]}"
+                name = f"[Linkgrabber] {details['name']}"
                 try:
                     mb = mb_left = int(details["bytesTotal"]) / (1024 * 1024)
                 except KeyError:
@@ -302,7 +312,7 @@ def get_packages(shared_state):
                 package_uuid = package["uuid"]
             else:
                 details = package["details"]
-                name = f"[CAPTCHA not solved!] {details["title"]}"
+                name = f"[CAPTCHA not solved!] {details['title']}"
                 mb = mb_left = details["size_mb"]
                 try:
                     package_id = package["package_id"]
@@ -386,23 +396,23 @@ def get_packages(shared_state):
         else:
             info(f"Invalid package location {package['location']}")
 
-    if not shared_state.get_device().linkgrabber.is_collecting():
-        linkgrabber_packages = shared_state.get_device().linkgrabber.query_packages()
-        linkgrabber_links = shared_state.get_device().linkgrabber.query_links()
-
+    # Use cached is_collecting check
+    if not cache.is_collecting:
+        # Reuse cached data instead of re-querying
         packages_to_start = []
         links_to_start = []
 
         for package in linkgrabber_packages:
-            comment = get_links_comment(package, shared_state.get_device().linkgrabber.query_links())
+            # Use cached linkgrabber_links instead of re-querying
+            comment = get_links_comment(package, linkgrabber_links)
             if comment and comment.startswith("Quasarr_"):
                 package_uuid = package.get("uuid")
                 if package_uuid:
-                    linkgrabber_links = [link.get("uuid") for link in linkgrabber_links if
-                                         link.get("packageUUID") == package_uuid]
-                    if linkgrabber_links:
+                    package_link_ids = [link.get("uuid") for link in linkgrabber_links if
+                                        link.get("packageUUID") == package_uuid]
+                    if package_link_ids:
                         packages_to_start.append(package_uuid)
-                        links_to_start.extend(linkgrabber_links)
+                        links_to_start.extend(package_link_ids)
                     else:
                         info(f"Package {package_uuid} has no links in linkgrabber - skipping start")
 
@@ -420,13 +430,17 @@ def delete_package(shared_state, package_id):
     try:
         deleted_title = ""
 
-        packages = get_packages(shared_state)
+        # Create cache for this single delete operation
+        # Safe to reuse within this request since we fetch->find->delete atomically
+        cache = JDPackageCache(shared_state.get_device())
+
+        packages = get_packages(shared_state, _cache=cache)
         for package_location in packages:
             for package in packages[package_location]:
                 if package["nzo_id"] == package_id:
                     if package["type"] == "linkgrabber":
-                        ids = get_links_matching_package_uuid(package,
-                                                              shared_state.get_device().linkgrabber.query_links())
+                        # Use cached linkgrabber_links instead of re-querying
+                        ids = get_links_matching_package_uuid(package, cache.linkgrabber_links)
                         if ids:
                             shared_state.get_device().linkgrabber.cleanup(
                                 "DELETE_ALL",
@@ -437,8 +451,8 @@ def delete_package(shared_state, package_id):
                             )
                             break
                     elif package["type"] == "downloader":
-                        ids = get_links_matching_package_uuid(package,
-                                                              shared_state.get_device().downloads.query_links())
+                        # Use cached downloader_links instead of re-querying
+                        ids = get_links_matching_package_uuid(package, cache.downloader_links)
                         if ids:
                             shared_state.get_device().downloads.cleanup(
                                 "DELETE_ALL",

quasarr/downloads/sources/dl.py

@@ -3,14 +3,12 @@
 # Project by https://github.com/rix1337
 
 import re
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from io import BytesIO
 
-from PIL import Image
 from bs4 import BeautifulSoup, NavigableString
 
 from quasarr.providers.log import info, debug
 from quasarr.providers.sessions.dl import retrieve_and_validate_session, fetch_via_requests_session, invalidate_session
+from quasarr.providers.utils import generate_status_url, check_links_online_status
 
 hostname = "dl"
 
@@ -136,28 +134,6 @@ def extract_mirror_name_from_link(link_element):
     return None
 
 
-def generate_status_url(href, crypter_type):
-    """
-    Generate a status URL for crypters that support it.
-    Returns None if status URL cannot be generated.
-    """
-    if crypter_type == "hide":
-        # hide.cx links: https://hide.cx/folder/{UUID} → https://hide.cx/state/{UUID}
-        match = re.search(r'hide\.cx/(?:folder/)?([a-f0-9-]{36})', href, re.IGNORECASE)
-        if match:
-            uuid = match.group(1)
-            return f"https://hide.cx/state/{uuid}"
-
-    elif crypter_type == "tolink":
-        # tolink links: https://tolink.to/f/{ID} → https://tolink.to/f/{ID}/s/status.png
-        match = re.search(r'tolink\.to/f/([a-zA-Z0-9]+)', href, re.IGNORECASE)
-        if match:
-            link_id = match.group(1)
-            return f"https://tolink.to/f/{link_id}/s/status.png"
-
-    return None
-
-
 def extract_status_url_from_html(link_element, crypter_type):
     """
     Extract status image URL from HTML near the link element.
@@ -259,100 +235,6 @@ def build_filecrypt_status_map(soup):
     return status_map
 
 
-def image_has_green(image_data):
-    """
-    Analyze image data to check if it contains green pixels.
-    Returns True if any significant green is detected (indicating online status).
-    """
-    try:
-        img = Image.open(BytesIO(image_data))
-        img = img.convert('RGB')
-
-        pixels = list(img.getdata())
-
-        for r, g, b in pixels:
-            # Check if pixel is greenish: green channel is dominant
-            # and has a reasonable absolute value
-            if g > 100 and g > r * 1.3 and g > b * 1.3:
-                return True
-
-        return False
-    except Exception as e:
-        debug(f"Error analyzing status image: {e}")
-        # If we can't analyze, assume online to not skip valid links
-        return True
-
-
-def fetch_status_image(status_url):
-    """
-    Fetch a status image and return (status_url, image_data).
-    Returns (status_url, None) on failure.
-    """
-    try:
-        import requests
-        response = requests.get(status_url, timeout=10)
-        if response.status_code == 200:
-            return (status_url, response.content)
-    except Exception as e:
-        debug(f"Error fetching status image {status_url}: {e}")
-    return (status_url, None)
-
-
-def check_links_online_status(links_with_status):
-    """
-    Check online status for links that have status URLs.
-    Returns list of links that are online (or have no status URL to check).
-
-    links_with_status: list of [href, identifier, status_url] where status_url can be None
-    """
-
-    links_to_check = [(i, link) for i, link in enumerate(links_with_status) if link[2]]
-
-    if not links_to_check:
-        # No status URLs to check, return all links as potentially online
-        return [[link[0], link[1]] for link in links_with_status]
-
-    # Batch fetch status images
-    status_results = {}  # status_url -> has_green
-    status_urls = list(set(link[2] for _, link in links_to_check))
-
-    batch_size = 10
-    for i in range(0, len(status_urls), batch_size):
-        batch = status_urls[i:i + batch_size]
-        with ThreadPoolExecutor(max_workers=batch_size) as executor:
-            futures = [executor.submit(fetch_status_image, url) for url in batch]
-            for future in as_completed(futures):
-                try:
-                    status_url, image_data = future.result()
-                    if image_data:
-                        status_results[status_url] = image_has_green(image_data)
-                    else:
-                        # Could not fetch, assume online
-                        status_results[status_url] = True
-                except Exception as e:
-                    debug(f"Error checking status: {e}")
-
-    # Filter to online links
-    online_links = []
-
-    for link in links_with_status:
-        href, identifier, status_url = link
-        if not status_url:
-            # No status URL, include link (keeplinks case)
-            online_links.append([href, identifier])
-        elif status_url in status_results:
-            if status_results[status_url]:
-                online_links.append([href, identifier])
-                debug(f"Link online: {identifier} ({href})")
-            else:
-                debug(f"Link offline: {identifier} ({href})")
-        else:
-            # Status check failed, include link
-            online_links.append([href, identifier])
-
-    return online_links
-
-
 def extract_links_and_password_from_post(post_content, host):
     """
     Extract download links and password from a forum post.
@@ -448,7 +330,11 @@ def get_dl_download_links(shared_state, url, mirror, title, password):
             info(f"Could not find any posts in thread: {url}")
             return {"links": [], "password": ""}
 
-        # Iterate through posts to find one with online links
+        # Track first post with unverifiable links as fallback
+        fallback_links = None
+        fallback_password = ""
+
+        # Iterate through posts to find one with verified online links
         for post_index, post in enumerate(posts):
             post_content = post.select_one('div.bbWrapper')
             if not post_content:
@@ -459,16 +345,32 @@ def get_dl_download_links(shared_state, url, mirror, title, password):
             if not links_with_status:
                 continue
 
+            # Check if any links have status URLs we can verify
+            has_verifiable_links = any(link[2] for link in links_with_status)
+
+            if not has_verifiable_links:
+                # No way to check online status - save as fallback and continue looking
+                if fallback_links is None:
+                    fallback_links = [[link[0], link[1]] for link in links_with_status]
+                    fallback_password = extracted_password
+                    debug(f"Post #{post_index + 1} has links but no status URLs, saving as fallback...")
+                continue
+
             # Check which links are online
-            online_links = check_links_online_status(links_with_status)
+            online_links = check_links_online_status(links_with_status, shared_state)
 
             if online_links:
                 post_info = "first post" if post_index == 0 else f"post #{post_index + 1}"
-                debug(f"Found {len(online_links)} online link(s) in {post_info} for: {title}")
+                debug(f"Found {len(online_links)} verified online link(s) in {post_info} for: {title}")
                 return {"links": online_links, "password": extracted_password}
             else:
                 debug(f"All links in post #{post_index + 1} are offline, checking next post...")
 
+        # No verified online links found - return fallback if available
+        if fallback_links:
+            debug(f"No verified online links found, returning unverified fallback links for: {title}")
+            return {"links": fallback_links, "password": fallback_password}
+
         info(f"No online download links found in any post: {url}")
         return {"links": [], "password": ""}
 

quasarr/downloads/sources/wx.py

@@ -7,6 +7,7 @@ import re
 import requests
 
 from quasarr.providers.log import info, debug
+from quasarr.providers.utils import check_links_online_status
 
 hostname = "wx"
 
@@ -15,7 +16,10 @@ def get_wx_download_links(shared_state, url, mirror, title, password):
     """
     KEEP THE SIGNATURE EVEN IF SOME PARAMETERS ARE UNUSED!
 
-    WX source handler - Grabs download links from API based on title and mirror.
+    WX source handler - Grabs download links from API based on title.
+    Finds the best mirror (M1, M2, M3...) by checking online status.
+    Returns all online links from the first complete mirror, or the best partial mirror.
+    Prefers hide.cx links over other crypters (filecrypt, etc.) when online counts are equal.
     """
     host = shared_state.values["config"]("Hostnames").get(hostname)
 
@@ -35,7 +39,7 @@ def get_wx_download_links(shared_state, url, mirror, title, password):
             return {"links": []}
 
         # Extract slug from URL
-        slug_match = re.search(r'/detail/([^/]+)', url)
+        slug_match = re.search(r'/detail/([^/?]+)', url)
         if not slug_match:
             info(f"{hostname.upper()}: Could not extract slug from URL: {url}")
             return {"links": []}
@@ -64,62 +68,100 @@ def get_wx_download_links(shared_state, url, mirror, title, password):
 
         releases = data['item']['releases']
 
-        # Find the release matching the title
-        matching_release = None
-        for release in releases:
-            if release.get('fulltitle') == title:
-                matching_release = release
-                break
+        # Find ALL releases matching the title (these are different mirrors: M1, M2, M3...)
+        matching_releases = [r for r in releases if r.get('fulltitle') == title]
 
-        if not matching_release:
+        if not matching_releases:
             info(f"{hostname.upper()}: No release found matching title: {title}")
             return {"links": []}
 
-        # Extract crypted_links based on mirror
-        crypted_links = matching_release.get('crypted_links', {})
-
-        if not crypted_links:
-            info(f"{hostname.upper()}: No crypted_links found for: {title}")
-            return {"links": []}
-
-        links = []
-
-        # If mirror is specified, find matching hoster (handle partial matches like 'ddownload' -> 'ddownload.com')
-        if mirror:
-            matched_hoster = None
-            for hoster in crypted_links.keys():
-                if mirror.lower() in hoster.lower() or hoster.lower() in mirror.lower():
-                    matched_hoster = hoster
-                    break
-
-            if matched_hoster:
-                link = crypted_links[matched_hoster]
-                # Prefer hide over filecrypt
-                if re.search(r'hide\.', link, re.IGNORECASE):
-                    links.append([link, matched_hoster])
-                    debug(f"{hostname.upper()}: Found hide link for mirror {matched_hoster}")
-                elif re.search(r'filecrypt\.', link, re.IGNORECASE):
-                    links.append([link, matched_hoster])
-                    debug(f"{hostname.upper()}: Found filecrypt link for mirror {matched_hoster}")
-            else:
-                info(
-                    f"{hostname.upper()}: Mirror '{mirror}' not found in available hosters: {list(crypted_links.keys())}")
-        else:
-            # If no mirror specified, get all available crypted links (prefer hide over filecrypt)
-            for hoster, link in crypted_links.items():
-                if re.search(r'hide\.', link, re.IGNORECASE):
-                    links.append([link, hoster])
-                    debug(f"{hostname.upper()}: Found hide link for hoster {hoster}")
-                elif re.search(r'filecrypt\.', link, re.IGNORECASE):
-                    links.append([link, hoster])
-                    debug(f"{hostname.upper()}: Found filecrypt link for hoster {hoster}")
-
-        if not links:
-            info(f"{hostname.upper()}: No supported crypted links found for: {title}")
-            return {"links": []}
-
-        debug(f"{hostname.upper()}: Found {len(links)} crypted link(s) for: {title}")
-        return {"links": links}
+        debug(f"{hostname.upper()}: Found {len(matching_releases)} mirror(s) for: {title}")
+
+        # Evaluate each mirror and find the best one
+        # Track: (online_count, is_hide, online_links)
+        best_mirror = None  # (online_count, is_hide, online_links)
+
+        for idx, release in enumerate(matching_releases):
+            crypted_links = release.get('crypted_links', {})
+            check_urls = release.get('options', {}).get('check', {})
+
+            if not crypted_links:
+                continue
+
+            # Separate hide.cx links from other crypters
+            hide_links = []
+            other_links = []
+
+            for hoster, container_url in crypted_links.items():
+                state_url = check_urls.get(hoster)
+                if re.search(r'hide\.', container_url, re.IGNORECASE):
+                    hide_links.append([container_url, hoster, state_url])
+                elif re.search(r'filecrypt\.', container_url, re.IGNORECASE):
+                    other_links.append([container_url, hoster, state_url])
+                # Skip other crypters we don't support
+
+            # Check hide.cx links first (preferred)
+            hide_online = 0
+            online_hide = []
+            if hide_links:
+                online_hide = check_links_online_status(hide_links, shared_state)
+                hide_total = len(hide_links)
+                hide_online = len(online_hide)
+
+                debug(f"{hostname.upper()}: M{idx + 1} hide.cx: {hide_online}/{hide_total} online")
+
+                # If all hide.cx links are online, use this mirror immediately
+                if hide_online == hide_total and hide_online > 0:
+                    debug(
+                        f"{hostname.upper()}: M{idx + 1} is complete (all {hide_online} hide.cx links online), using this mirror")
+                    return {"links": online_hide}
+
+            # Check other crypters (filecrypt, etc.) - no early return, always check all mirrors for hide.cx first
+            other_online = 0
+            online_other = []
+            if other_links:
+                online_other = check_links_online_status(other_links, shared_state)
+                other_total = len(other_links)
+                other_online = len(online_other)
+
+                debug(f"{hostname.upper()}: M{idx + 1} other crypters: {other_online}/{other_total} online")
+
+            # Determine best option for this mirror (prefer hide.cx on ties)
+            mirror_links = None
+            mirror_count = 0
+            mirror_is_hide = False
+
+            if hide_online > 0 and hide_online >= other_online:
+                # hide.cx wins (more links or tie)
+                mirror_links = online_hide
+                mirror_count = hide_online
+                mirror_is_hide = True
+            elif other_online > hide_online:
+                # other crypter has more online links
+                mirror_links = online_other
+                mirror_count = other_online
+                mirror_is_hide = False
+
+            # Update best_mirror if this mirror is better
+            # Priority: 1) more online links, 2) hide.cx preference on ties
+            if mirror_links:
+                if best_mirror is None:
+                    best_mirror = (mirror_count, mirror_is_hide, mirror_links)
+                elif mirror_count > best_mirror[0]:
+                    best_mirror = (mirror_count, mirror_is_hide, mirror_links)
+                elif mirror_count == best_mirror[0] and mirror_is_hide and not best_mirror[1]:
+                    # Same count but this is hide.cx and current best is not
+                    best_mirror = (mirror_count, mirror_is_hide, mirror_links)
+
+        # No complete mirror found, return best partial mirror
+        if best_mirror and best_mirror[2]:
+            crypter_type = "hide.cx" if best_mirror[1] else "other crypter"
+            debug(
+                f"{hostname.upper()}: No complete mirror, using best partial with {best_mirror[0]} online {crypter_type} link(s)")
+            return {"links": best_mirror[2]}
+
+        info(f"{hostname.upper()}: No online links found for: {title}")
+        return {"links": []}
 
     except Exception as e:
         info(f"{hostname.upper()}: Error extracting download links from {url}: {e}")