cyberdrop-dl-patched 5.7.2.dev2__tar.gz → 5.7.2.dev4__tar.gz

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/CHANGELOG.md

@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 This update introduces the following changes:
 
-1. Adds the option to limit how many items are scraped 
+1. Adds the option to limit how many items are scraped
 2. Add support for scraping a users' coomer favorites
 3. Add integration to handle downloads supported by https://real-debrid.com
 4. Add support for https://nekohouse.su profiles and posts
@@ -29,7 +29,7 @@ This update introduces the following changes:
 
 - Users can limit the number of items to scrape by type, using the `--maximum-number-of-children` parameter. For more details on how to use this feature, visit the wiki: https://script-ware.gitbook.io/cyberdrop-dl/reference/configuration-options/settings#download-options
 - Add support for scraping a users' coomer favorites by allowing the user to pass the coomer favorites page URL as an input URL (https://coomer.su/favorites). This requires them to have their coomer session token in the `authentication.yaml` file.
-- Add real-debrid integration to download from any site that they support (`mega.nz`,`rapidgator`, `google drive`, `k2s`, etc). User needs to provide their API key in the `authentication.yaml` file in order to allow downloads 
+- Add real-debrid integration to download from any site that they support (`mega.nz`,`rapidgator`, `google drive`, `k2s`, etc). User needs to provide their API key in the `authentication.yaml` file in order to allow downloads
 - Nekohouse URLs can now be scraped and downloaded by CDL
 - Users can now get the stats report of the run via multiple services and include the main log as an attachment. For more information on how to setup notifications, visit: https://script-ware.gitbook.io/cyberdrop-dl/reference/notifications
 - Fix parsing of bunkr file extensions when `--remove-generated-id` is enabled
@@ -41,7 +41,7 @@ This update introduces the following changes:
 ## [5.7.0] - 2024-10-25
 
 This update introduces the following changes:
-1. Rotating log files 
+1. Rotating log files
 2. Overhaul hashing functions
 3. Add support for https://tokyomotion.net URLs
 4. Add support for https://xxxbunker.com URLs
@@ -69,7 +69,7 @@ This update introduces the following changes:
 - Fix error during program exit when referers table no longer exists
 - Prevents crashes when there are insufficient permissions to move a file
 - Fix an issue where CDL would delete URLs input file
-- Move functions for after download to `post_runtime` 
+- Move functions for after download to `post_runtime`
 - Fix handling of deleted imgbox albums if they return `HTTP 200`
 - Fix cyberfile deleted folders not being correctly scraped
 - Update logging to include when a file is being downloaded with no crawler
@@ -164,7 +164,7 @@ This update introduces the following changes:
 ## [5.6.40] - 2024-10-01
 
 This update introduces the following changes:
-1. Fixes empty folder cleanup 
+1. Fixes empty folder cleanup
 
 #### Details:
 - Fixes incorrent path objects on post-runtime folder cleanup

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cyberdrop-dl-patched
-Version: 5.7.2.dev2
+Version: 5.7.2.dev4
 Summary: Bulk downloader for multiple file hosts
 Home-page: https://github.com/jbsparrow/CyberDropDownloader
 Author: Jacob B

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/cyberdrop_dl/clients/download_client.py

@@ -76,12 +76,12 @@ class DownloadClient:
     def add_request_log_hooks(self) -> None:
         async def on_request_start(*args):
             params: aiohttp.TraceRequestStartParams = args[2]
-            log(f"Starting download {params.method} request to {params.url}", 40)
+            log(f"Starting download {params.method} request to {params.url}", 20)
 
         async def on_request_end(*args):
             params: aiohttp.TraceRequestEndParams = args[2]
-            log(f"Finishing download {params.method} request to {params.url}", 40)
-            log(f"Response status for {params.url}: {params.response.status}", 40)
+            log(f"Finishing download {params.method} request to {params.url}", 20)
+            log(f"Response status for {params.url}: {params.response.status}", 20)
 
         trace_config = aiohttp.TraceConfig()
         trace_config.on_request_start.append(on_request_start)

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/cyberdrop_dl/clients/scraper_client.py

@@ -65,12 +65,12 @@ class ScraperClient:
     def add_request_log_hooks(self) -> None:
         async def on_request_start(*args):
             params: aiohttp.TraceRequestStartParams = args[2]
-            log(f"Starting download {params.method} request to {params.url}", 40)
+            log(f"Starting download {params.method} request to {params.url}", 20)
 
         async def on_request_end(*args):
             params: aiohttp.TraceRequestEndParams = args[2]
-            log(f"Finishing download {params.method} request to {params.url}", 40)
-            log(f"Response status for {params.url}: {params.response.status}", 40)
+            log(f"Finishing download {params.method} request to {params.url}", 20)
+            log(f"Response status for {params.url}: {params.response.status}", 20)
 
         trace_config = aiohttp.TraceConfig()
         trace_config.on_request_start.append(on_request_start)

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/cyberdrop_dl/downloader/downloader.py

@@ -144,8 +144,6 @@ class Downloader:
                     log(f"{self.log_prefix} failed: {media_item.url} with error {e}", 40, exc_info=True)
                     self.manager.progress_manager.download_stats_progress.add_failure("Unknown")
                     self.manager.progress_manager.download_progress.add_failed()
-                else:
-                    log(f"{self.log_prefix} finished: {media_item.url}", 20)
                 finally:
                     await self._file_lock.release_lock(media_item.file_lock_reference_name)
         self._semaphore.release()
@@ -195,6 +193,7 @@ class Downloader:
                 self.set_file_datetime(media_item, media_item.complete_file)
                 self.attempt_task_removal(media_item)
                 self.manager.progress_manager.download_progress.add_completed()
+                log(f"Download finished: {media_item.url}", 20)
 
         except RestrictedFiletypeError:
             self.manager.progress_manager.download_progress.add_skipped()

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/cyberdrop_dl/main.py

@@ -5,8 +5,11 @@ import contextlib
 import logging
 import os
 import sys
+from functools import wraps
 from pathlib import Path
+from textwrap import indent
 from time import perf_counter
+from typing import TYPE_CHECKING
 
 from rich.console import Console
 from rich.logging import RichHandler
@@ -15,6 +18,7 @@ from cyberdrop_dl.clients.errors import InvalidYamlError
 from cyberdrop_dl.managers.manager import Manager
 from cyberdrop_dl.scraper.scraper import ScrapeMapper
 from cyberdrop_dl.ui.ui import program_ui
+from cyberdrop_dl.utils.args.browser_cookie_extraction import get_cookies_from_browser
 from cyberdrop_dl.utils.logger import (
     log,
     log_spacer,
@@ -29,6 +33,9 @@ from cyberdrop_dl.utils.utilities import (
     sent_apprise_notifications,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 def startup() -> Manager:
     """Starts the program and returns the manager.
@@ -67,6 +74,12 @@ async def runtime(manager: Manager) -> None:
             await scrape_mapper.start()
 
 
+def pre_runtime(manager: Manager) -> None:
+    """Actions to complete before main runtime."""
+    if manager.config_manager.settings_data["Browser_Cookies"]["auto_import"]:
+        get_cookies_from_browser(manager)
+
+
 async def post_runtime(manager: Manager) -> None:
     """Actions to complete after main runtime, and before ui shutdown."""
     log_spacer(20)
@@ -158,6 +171,25 @@ def setup_logger(manager: Manager, config_name: str) -> None:
     constants.CONSOLE_LEVEL = manager.config_manager.settings_data["Runtime_Options"]["console_log_level"]
 
 
+def ui_error_handling_wrapper(func: Callable) -> None:
+    """Wrapper handles errors from the main UI"""
+
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except* Exception as e:
+            exc_list = str(e)
+            if isinstance(e, ExceptionGroup):
+                exc_list = "\n".join(map(str, e.exceptions))
+            exc_list = indent(exc_list, "  ")
+            msg = f"An error occurred, please report this to the developer with your logs file:\n{exc_list}"
+            log_with_color(msg, "bold red", 50, show_in_stats=False, exc_info=e)
+
+    return wrapper
+
+
+@ui_error_handling_wrapper
 async def director(manager: Manager) -> None:
     """Runs the program and handles the UI."""
     manager.path_manager.startup()
@@ -182,18 +214,9 @@ async def director(manager: Manager) -> None:
         log_spacer(20)
         log("Starting CDL...\n", 20)
 
-        try:
-            await runtime(manager)
-            await post_runtime(manager)
-        except* Exception as e:
-            log_with_color(
-                f"An error occurred, please report this to the developer: {e}",
-                "bold red",
-                50,
-                show_in_stats=False,
-                exc_info=True,
-            )
-            sys.exit(1)
+        pre_runtime(manager)
+        await runtime(manager)
+        await post_runtime(manager)
 
         log_spacer(20)
         manager.progress_manager.print_stats(start_time)
@@ -204,7 +227,6 @@ async def director(manager: Manager) -> None:
             check_latest_pypi()
             log_spacer(20)
             log("Closing Program...", 20)
-            await manager.close()
             log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)
 
         await send_webhook_message(manager)
@@ -217,18 +239,14 @@ def main() -> None:
     asyncio.set_event_loop(loop)
     exit_code = 1
     manager = startup()
-    with contextlib.suppress(RuntimeError):
+    with contextlib.suppress(Exception):
         try:
             asyncio.run(director(manager))
             exit_code = 0
         except KeyboardInterrupt:
             print_to_console("Trying to Exit ...")
-            with contextlib.suppress(Exception):
-                asyncio.run(manager.close())
-            sys.exit(1)
-    loop.close()
-    sys.exit(0)
-
+        finally:
+            asyncio.run(manager.close())
     loop.close()
     sys.exit(exit_code)
 

{cyberdrop_dl_patched-5.7.2.dev2 → cyberdrop_dl_patched-5.7.2.dev4}/cyberdrop_dl/managers/client_manager.py

@@ -8,18 +8,19 @@ from typing import TYPE_CHECKING
 
 import aiohttp
 import certifi
-from aiohttp import ClientResponse, ContentTypeError
+from aiohttp import ClientResponse, ClientSession, ContentTypeError
 from aiolimiter import AsyncLimiter
+from bs4 import BeautifulSoup
+from yarl import URL
 
 from cyberdrop_dl.clients.download_client import DownloadClient
 from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, ScrapeError
 from cyberdrop_dl.clients.scraper_client import ScraperClient
 from cyberdrop_dl.managers.leaky import LeakyBucket
 from cyberdrop_dl.utils.constants import CustomHTTPStatus
+from cyberdrop_dl.utils.logger import log
 
 if TYPE_CHECKING:
-    from yarl import URL
-
     from cyberdrop_dl.managers.manager import Manager
     from cyberdrop_dl.scraper.crawler import ScrapeItem
 
@@ -28,6 +29,18 @@ DOWNLOAD_ERROR_ETAGS = {
     "65b7753c-528a": "SC Scrape Image",
 }
 
+DDOS_GUARD_CHALLENGE_TITLES = ["Just a moment...", "DDoS-Guard"]
+DDOS_GUARD_CHALLENGE_SELECTORS = [
+    "#cf-challenge-running",
+    ".ray_id",
+    ".attack-box",
+    "#cf-please-wait",
+    "#challenge-spinner",
+    "#trk_jschal_js",
+    "#turnstile-wrapper",
+    ".lds-ring",
+]
+
 
 class ClientManager:
     """Creates a 'client' that can be referenced by scraping or download sessions."""
@@ -54,11 +67,6 @@ class ClientManager:
             if not manager.args_manager.proxy
             else manager.args_manager.proxy
         )
-        self.flaresolverr = (
-            manager.config_manager.global_settings_data["General"]["flaresolverr"]
-            if not manager.args_manager.flaresolverr
-            else manager.args_manager.flaresolverr
-        )
 
         self.domain_rate_limits = {
             "bunkrr": AsyncLimiter(5, 1),
@@ -88,6 +96,7 @@ class ClientManager:
         self.scraper_session = ScraperClient(self)
         self.downloader_session = DownloadClient(manager, self)
         self._leaky_bucket = LeakyBucket(manager)
+        self.flaresolverr = Flaresolverr(self)
 
     """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
 
@@ -105,8 +114,9 @@ class ClientManager:
 
     """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
 
-    @staticmethod
+    @classmethod
     async def check_http_status(
+        cls,
         response: ClientResponse,
         download: bool = False,
         origin: ScrapeItem | URL | None = None,
@@ -130,9 +140,13 @@ class ClientManager:
                 if "data" in JSON_Resp and "error" in JSON_Resp["data"]:
                     raise ScrapeError(JSON_Resp["status"], JSON_Resp["data"]["error"], origin=origin)
 
+        response_text = None
         with contextlib.suppress(UnicodeDecodeError):
             response_text = await response.text()
-            if "<title>DDoS-Guard</title>" in response_text:
+
+        if response_text:
+            soup = BeautifulSoup(response_text, "html.parser")
+            if cls.check_ddos_guard(soup):
                 raise DDOSGuardError(origin=origin)
 
         status = status if headers.get("Content-Type") else CustomHTTPStatus.IM_A_TEAPOT
@@ -147,3 +161,126 @@ class ClientManager:
 
     async def check_bucket(self, size: float) -> None:
         await self._leaky_bucket.acquire(size)
+
+    @staticmethod
+    def check_ddos_guard(soup: BeautifulSoup) -> bool:
+        if soup.title:
+            for title in DDOS_GUARD_CHALLENGE_TITLES:
+                challenge_found = title.casefold() == soup.title.string.casefold()
+                if challenge_found:
+                    return True
+
+        for selector in DDOS_GUARD_CHALLENGE_SELECTORS:
+            challenge_found = soup.find(selector)
+            if challenge_found:
+                return True
+
+        return False
+
+    async def close(self) -> None:
+        await self.flaresolverr._destroy_session()
+
+
+class Flaresolverr:
+    """Class that handles communication with flaresolverr."""
+
+    def __init__(self, client_manager: ClientManager) -> None:
+        self.client_manager = client_manager
+        self.flaresolverr_host = (
+            client_manager.manager.args_manager.flaresolverr
+            or client_manager.manager.config_manager.global_settings_data["General"]["flaresolverr"]
+        )
+        self.enabled = bool(self.flaresolverr_host)
+        if "http" not in self.flaresolverr_host:
+            self.flaresolverr_host = f"http://{self.flaresolverr_host}"
+        self.session_id = None
+        self.flaresolverr_host = URL(self.flaresolverr_host)
+
+    async def _request(
+        self,
+        command: str,
+        client_session: ClientSession,
+        origin: ScrapeItem | URL | None = None,
+        **kwargs,
+    ) -> dict:
+        """Base request function to call flaresolverr."""
+        if not self.enabled:
+            raise DDOSGuardError(message="FlareSolverr is not configured", origin=origin)
+
+        if not (self.session_id or kwargs.get("session")):
+            await self._create_session()
+
+        headers = client_session.headers.copy()
+        headers.update({"Content-Type": "application/json"})
+        for key, value in kwargs.items():
+            if isinstance(value, URL):
+                kwargs[key] = str(value)
+
+        data = {"cmd": command, "maxTimeout": 60000, "session": self.session_id} | kwargs
+
+        async with client_session.post(
+            self.flaresolverr_host / "v1",
+            headers=headers,
+            ssl=self.client_manager.ssl_context,
+            proxy=self.client_manager.proxy,
+            json=data,
+        ) as response:
+            json_obj: dict = await response.json()  # type: ignore
+
+        return json_obj
+
+    async def _create_session(self) -> None:
+        """Creates a permanet flaresolverr session."""
+        session_id = "cyberdrop-dl"
+        async with ClientSession() as client_session:
+            flaresolverr_resp = await self._request("sessions.create", client_session, session=session_id)
+        status = flaresolverr_resp.get("status")
+        if status != "ok":
+            raise DDOSGuardError(message="Failed to create flaresolverr session")
+        self.session_id = session_id
+
+    async def _destroy_session(self):
+        if self.session_id:
+            async with ClientSession() as client_session:
+                await self._request("sessions.destroy", client_session, session=self.session_id)
+
+    async def get(
+        self,
+        url: URL,
+        client_session: ClientSession,
+        origin: ScrapeItem | URL | None = None,
+        update_cookies: bool = True,
+    ) -> tuple[BeautifulSoup, URL]:
+        """Returns the resolved URL from the given URL."""
+        flaresolverr_resp: dict = await self._request("request.get", client_session, origin, url=url)
+
+        status = flaresolverr_resp.get("status")
+        if status != "ok":
+            raise DDOSGuardError(message="Failed to resolve URL with flaresolverr", origin=origin)
+
+        solution: dict = flaresolverr_resp.get("solution")
+        if not solution:
+            raise DDOSGuardError(message="Invalid response from flaresolverr", origin=origin)
+        response = BeautifulSoup(solution.get("response"), "html.parser")
+        response_url = URL(solution.get("url"))
+        cookies: list[dict] = solution.get("cookies")
+        user_agent = client_session.headers.get("User-Agent").strip()
+        flaresolverr_user_agent = solution.get("userAgent").strip()
+        mismatch_msg = f"Config user_agent and flaresolverr user_agent do not match:\n  Cyberdrop-DL: {user_agent}\n  Flaresolverr: {flaresolverr_user_agent}"
+
+        if self.client_manager.check_ddos_guard(response):
+            if not update_cookies:
+                raise DDOSGuardError(message="Invalid response from flaresolverr", origin=origin)
+            if flaresolverr_user_agent != user_agent:
+                raise DDOSGuardError(message=mismatch_msg, origin=origin)
+
+        if update_cookies:
+            if flaresolverr_user_agent != user_agent:
+                log(f"{mismatch_msg}\nResponse was successful but cookies will not be valid", 30)
+
+            for cookie in cookies:
+                self.client_manager.cookies.update_cookies(
+                    {cookie["name"]: cookie["value"]}, URL(f"https://{cookie['domain']}")
+                )
+
+        return response, response_url