web-novel-scraper 1.1.0__tar.gz → 1.1.1__tar.gz

{web_novel_scraper-1.1.0 → web_novel_scraper-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-novel-scraper
-Version: 1.1.0
+Version: 1.1.1
 Summary: Python tool that allows you to scrape web novels from various sources and save them to more readable formats like EPUB.
 Project-URL: Homepage, https://github.com/ImagineBrkr/web-novel-scraper
 Project-URL: Documentation, https://web-novel-scraper.readthedocs.io

{web_novel_scraper-1.1.0 → web_novel_scraper-1.1.1}/web_novel_scraper/decode.py

@@ -16,6 +16,13 @@ DECODE_GUIDE_FILE = os.getenv('DECODE_GUIDE_FILE', f'{CURRENT_DIR}/decode_guide/
 
 XOR_SEPARATOR = "XOR"
 
+DEFAULT_REQUEST_CONFIG = {
+    "force_flaresolver": False,
+    "request_retries": 3,
+    "request_timeout": 20,
+    "request_time_between_retries": 3
+}
+
 try:
     with open(DECODE_GUIDE_FILE, 'r', encoding='UTF-8') as f:
         DECODE_GUIDE = json.load(f)
@@ -36,11 +43,30 @@ except Exception as e:
 class Decoder:
     host: str
     decode_guide: json
+    request_config: dict
 
     def __init__(self, host: str):
         self.host = host
         self.decode_guide = self._get_element_by_key(
             DECODE_GUIDE, 'host', host)
+        host_request_config = self.get_request_config()
+        self.request_config = DEFAULT_REQUEST_CONFIG | host_request_config
+
+    def get_request_config(self) -> dict:
+        request_config = self.decode_guide.get('request_config')
+        if request_config:
+            logger.debug(f'Host "{self.host}" has a custom request configuration on the Decode Guide file.')
+            return request_config
+
+        return DEFAULT_REQUEST_CONFIG
+
+    def is_index_inverted(self, host:str = None) -> bool:
+        if host:
+            decode_guide = self._get_element_by_key(DECODE_GUIDE, 'host', host)
+        else:
+            decode_guide = self.decode_guide
+
+        return decode_guide.get('index', {}).get('inverted', False)
 
     def get_chapter_urls(self, html: str) -> list[str]:
         logger.debug('Obtaining chapter URLs...')
@@ -130,7 +156,7 @@ class Decoder:
             return ' '.join(elements)
         return elements
 
-    def has_pagination(self, host: str = None):
+    def has_pagination(self, host: str = None) -> bool:
         if host:
             decode_guide = self._get_element_by_key(DECODE_GUIDE, 'host', host)
             return decode_guide['has_pagination']
@@ -245,9 +271,8 @@ class Decoder:
         return elements[0]
 
     @staticmethod
-    def _get_element_by_key(json_data, key, value):
+    def _get_element_by_key(json_data, key: str, value: str):
         for item in json_data:
             if item[key] == value:
                 return item
-        logger.warning('Host not found, using default decoder.')
         return json_data[0]

{web_novel_scraper-1.1.0 → web_novel_scraper-1.1.1}/web_novel_scraper/decode_guide/decode_guide.json

@@ -160,6 +160,9 @@
     },
     {
         "host": "novelbin.com",
+        "request_config": {
+            "force_flaresolver": "true"
+        },
         "has_pagination": false,
         "title": {
             "element": "h2 a.chr-title",
@@ -295,5 +298,40 @@
                 "key": "href"
             }
         }
+    },
+    {
+        "host": "scribblehub.com",
+        "request_config": {
+            "force_flaresolver": "true",
+            "request_timeout": 60
+        },
+        "has_pagination": true,
+        "title": {
+            "selector": "div.chapter-title",
+            "extract": {
+                "type": "text"
+            }
+        },
+        "content": {
+            "selector": "div.chp_raw p",
+            "array": true
+        },
+        "index": {
+            "selector": "div.toc ol li a",
+            "array": true,
+            "inverted": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
+        },
+        "next_page": {
+            "selector": "div ul.simple-pagination li a.next",
+            "array": false,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
+        }
     }
 ]

{web_novel_scraper-1.1.0 → web_novel_scraper-1.1.1}/web_novel_scraper/novel_scraper.py

@@ -10,7 +10,7 @@ from .decode import Decoder
 from .file_manager import FileManager
 from . import utils
 
-from . import request_manager
+from .request_manager import get_html_content
 
 logger = logger_manager.create_logger('NOVEL SCRAPPING')
 
@@ -277,8 +277,16 @@ class Novel:
             if chapters_url_from_toc_content is None:
                 logger.error('Chapters url not found on toc_content')
                 return False
-            self.chapters_url_list = [*self.chapters_url_list,
-                                      *chapters_url_from_toc_content]
+                # First we save a list of lists in case we need to invert the orderAdd commentMore actions
+            self.chapters_url_list.append(chapters_url_from_toc_content)
+
+        invert = self.decoder.is_index_inverted()
+        self.chapters_url_list = [
+            chapter
+            for chapters_url in (self.chapters_url_list[::-1] if invert else self.chapters_url_list)
+            for chapter in chapters_url
+        ]
+
         if self.scraper_behavior.auto_add_host:
             self.chapters_url_list = [
                 f'https://{self.host}{chapter_url}' for chapter_url in self.chapters_url_list]
@@ -464,6 +472,16 @@ class Novel:
             toc = self.decoder.clean_html(toc, hard_clean=hard_clean)
             self.file_manager.update_toc(toc, i)
 
+    def _request_html_content(self, url: str) -> Optional[str]:
+        request_config = self.decoder.request_config
+        force_flaresolver = request_config.get('force_flaresolver') or self.scraper_behavior.force_flaresolver
+        html_content = get_html_content(url,
+                                        retries=request_config.get('request_retries'),
+                                        timeout=request_config.get('request_timeout'),
+                                        time_between_retries=request_config.get('request_time_between_retries'),
+                                        force_flaresolver=force_flaresolver)
+        return html_content
+
     def _get_chapter(self,
                      chapter: Chapter,
                      reload: bool = False) -> Chapter | None:
@@ -481,8 +499,7 @@ class Novel:
                 return chapter
 
         # Fetch fresh content
-        chapter.chapter_html = request_manager.get_html_content(chapter.chapter_url,
-                                                                force_flaresolver=self.scraper_behavior.force_flaresolver)
+        chapter.chapter_html = self._request_html_content(chapter.chapter_url)
         if not chapter.chapter_html:
             logger.error(f'No content found on link {chapter.chapter_url}')
             return chapter
@@ -501,7 +518,11 @@ class Novel:
             if content:
                 return content
 
-        content = request_manager.get_html_content(url)
+        if utils.check_incomplete_url(url):
+            url = self.toc_main_url + url
+
+        # Fetch fresh content
+        content = self._request_html_content(url)
         if not content:
             logger.warning(f'No content found on link {url}')
             sys.exit(1)

web_novel_scraper-1.1.1/web_novel_scraper/request_manager.py

@@ -0,0 +1,147 @@
+import requests
+import os
+from . import logger_manager
+from dotenv import load_dotenv
+import json
+import time
+from typing import Optional
+
+load_dotenv()
+
+FLARESOLVER_URL = os.getenv('SCRAPER_FLARESOLVER_URL', 'http://localhost:8191/v1')
+FLARE_HEADERS = {'Content-Type': 'application/json'}
+FORCE_FLARESOLVER = os.getenv('FORCE_FLARESOLVER', '0') == '1'
+
+logger = logger_manager.create_logger('GET HTML CONTENT')
+
+
+def _get_request(url: str,
+                 timeout: int,
+                 retries: int,
+                 time_between_retries: int) -> Optional[requests.Response]:
+    logger.debug(
+        f'Starting get_request for "{url}" with timeout={timeout}, '
+        f'retries={retries}, '
+        f'time_between_retries={time_between_retries}')
+    for attempt in range(retries):
+        logger.debug(f'Attempt {attempt + 1} for "{url}"')
+        try:
+            response = requests.get(url, timeout=timeout)
+            response.raise_for_status()
+            logger.debug(f'Successful response for "{url}" on attempt {attempt + 1}')
+            return response
+        except requests.exceptions.ConnectionError as e:
+            logger.debug(f'Connection error ({attempt + 1}/{retries}): {e}')
+        except requests.exceptions.Timeout as e:
+            logger.debug(f'Request timed out ({attempt + 1}/{retries}): {e}')
+        except requests.exceptions.HTTPError as e:
+            logger.debug(f'HTTP error ({attempt + 1}/{retries}): {e}')
+        except requests.exceptions.InvalidSchema as e:
+            logger.debug(f'Invalid URL schema for "{url}": {e}')
+            break  # Don't retry on invalid schema
+        except requests.exceptions.RequestException as e:
+            logger.debug(f'Request failed ({attempt + 1}/{retries}): {e}')
+
+        if attempt < retries - 1:
+            logger.debug(f'Waiting {time_between_retries} seconds before retrying')
+            time.sleep(time_between_retries)  # Wait before retrying
+    logger.warning(f'Failed to get a successful response for "{url}" after {retries} attempts using common HTTP Request')
+    return None
+
+
+def _get_request_flaresolver(url: str,
+                             timeout: int,
+                             retries: int,
+                             time_between_retries: int,
+                             flaresolver_url: str) -> Optional[requests.Response]:
+    logger.debug(
+        f'Starting get_request_flaresolver for "{url}" with timeout={timeout}, '
+        f'retries={retries}, '
+        f'time_between_retries={time_between_retries}')
+    for attempt in range(retries):
+        logger.debug(f'Attempt {attempt + 1} for "{url}" using FlareSolver')
+        try:
+            response = requests.post(
+                flaresolver_url,
+                headers=FLARE_HEADERS,
+                json={
+                    'cmd': 'request.get',
+                    'url': url,
+                    'maxTimeout': timeout * 1000
+                },
+                timeout=timeout
+            )
+            response.raise_for_status()
+            logger.debug(f'Successful response for "{url}" on attempt {attempt + 1} using FlareSolver')
+            return response
+
+        except requests.exceptions.ConnectionError as e:
+            logger.warning(f'Connection error with flaresolver (URL: "{flaresolver_url}"): {e}')
+            logger.warning(f'If the url is incorrect, set the env variable "FLARESOLVER_URL" to the correct value')
+            logger.warning('If FlareSolver is not installed in your machine, consider installing it.')
+            break # Don't retry on Connection Error
+        except requests.exceptions.Timeout as e:
+            logger.debug(f'Request timed out ({attempt + 1}/{retries}): {e}')
+        except requests.exceptions.InvalidSchema as e:
+            logger.debug(f'Invalid FlareSolver URL schema "{flaresolver_url}": {e}')
+            break  # Don't retry on invalid schema
+        except requests.exceptions.HTTPError as e:
+            logger.debug(f'HTTP error ({attempt + 1}/{retries}): {e}')
+        except requests.exceptions.RequestException as e:
+            logger.debug(f'Request failed ({attempt + 1}/{retries}): {e}')
+        except json.JSONDecodeError as e:
+            logger.debug(f'Invalid JSON response ({attempt + 1}/{retries}): {e}')
+
+        if attempt < retries - 1:
+            logger.debug(f'Waiting {time_between_retries} seconds before retrying')
+            time.sleep(time_between_retries)  # Wait before retrying
+
+    logger.warning(f'Failed to get a successful response for "{url}" using FlareSolver after {retries} attempts')
+    return None
+
+
+def get_html_content(url: str,
+                     retries: int = 3,
+                     timeout: int = 20,
+                     time_between_retries: int = 3,
+                     flaresolver_url: str = FLARESOLVER_URL,
+                     force_flaresolver: bool = FORCE_FLARESOLVER) -> Optional[str]:
+    logger.debug(
+        f'Requesting HTML Content for "{url}" with '
+        f'retries: "{retries}", '
+        f'timeout: "{timeout}", '
+        f'time between retries: "{time_between_retries}"')
+    if force_flaresolver:
+        logger.debug('Will directly try with FlareSolver')
+
+    # First try with common HTTP request
+    if not force_flaresolver:
+        response = _get_request(url,
+                                timeout=timeout,
+                                retries=retries,
+                                time_between_retries=time_between_retries)
+        if response and response.ok:
+            logger.debug(f'Successfully retrieved HTML content from "{url}" using common HTTP request')
+            return response.text
+
+    # Try with Flaresolver
+    logger.debug(f'Trying with Flaresolver for "{url}"')
+    response = _get_request_flaresolver(url,
+                                timeout=timeout,
+                                retries=retries,
+                                time_between_retries=time_between_retries,
+                                flaresolver_url=flaresolver_url)
+    if not response or not response.ok:
+        logger.warning(f'Failed all attempts to get HTML content from "{url}')
+        return None
+
+    response_json = response.json()
+    if 'solution' not in response_json:
+        logger.warning(f'No solution found in FlareSolver response for "{url}"')
+        return None
+    if 'response' not in response_json['solution']:
+        logger.warning(f'No response found in FlareSolver solution for "{url}"')
+        return None
+
+    logger.debug(f'Successfully retrieved HTML content from "{url}" using FlareSolver')
+    return response_json['solution']['response']

{web_novel_scraper-1.1.0 → web_novel_scraper-1.1.1}/web_novel_scraper/utils.py

@@ -64,3 +64,10 @@ def check_exclusive_params(param1: any, param2: any) -> bool:
 
 def create_volume_id(n: int):
     return f'v{n:02}'
+
+def check_incomplete_url(url: str) -> bool:
+    if url.startswith('?') or url.startswith('#'):
+        return True
+
+    parsed = urlparse(url)
+    return not parsed.scheme or not parsed.netloc

web_novel_scraper-1.1.1/web_novel_scraper/version.py

@@ -0,0 +1 @@
+__version__ = "1.1.1"

web_novel_scraper-1.1.0/web_novel_scraper/request_manager.py

@@ -1,134 +0,0 @@
-import requests
-import os
-from . import logger_manager
-from dotenv import load_dotenv
-import json
-import time
-
-load_dotenv()
-
-FLARESOLVER_URL = os.getenv('SCRAPER_FLARESOLVER_URL', 'http://localhost:8191/v1')
-FLARE_HEADERS = {'Content-Type': 'application/json'}
-FORCE_FLARESOLVER = os.getenv('FORCE_FLARESOLVER', '0') == '1'
-
-logger = logger_manager.create_logger('GET HTML CONTENT')
-
-def get_request(url: str,
-                timeout: int = 20,
-                retries: int = 3,
-                time_between_retries: int = 1) -> requests.Response | None:
-    logger.debug(f'Starting get_request for {url} with timeout={timeout}, retries={retries}, time_between_retries={time_between_retries}')
-    for attempt in range(retries):
-        logger.debug(f'Attempt {attempt + 1} for {url}')
-        try:
-            response = requests.get(url, timeout=timeout)
-            response.raise_for_status()
-            logger.debug(f'Successful response for {url} on attempt {attempt + 1}')
-            return response
-        except requests.exceptions.ConnectionError as e:
-            logger.error(f'Connection error ({attempt + 1}/{retries}): {e}')
-        except requests.exceptions.Timeout as e:
-            logger.error(f'Request timed out ({attempt + 1}/{retries}): {e}')
-        except requests.exceptions.HTTPError as e:
-            logger.error(f'HTTP error ({attempt + 1}/{retries}): {e}')
-        except requests.exceptions.InvalidSchema as e:
-            logger.error(f'Invalid URL schema for "{url}": {e}')
-            break  # Don't retry on invalid schema
-        except requests.exceptions.RequestException as e:
-            logger.error(f'Request failed ({attempt + 1}/{retries}): {e}')
-
-        if attempt < retries - 1:
-            logger.debug(f'Waiting {time_between_retries} seconds before retrying')
-            time.sleep(time_between_retries)  # Wait before retrying
-    logger.debug(f'Failed to get a successful response for {url} after {retries} attempts')
-    return None
-
-
-def get_request_flaresolver(url: str,
-                            timeout: int = 20,
-                            flaresolver_url: str = FLARESOLVER_URL,
-                            retries: int = 3,
-                            time_between_retries: int = 1) -> requests.Response | None:
-    logger.debug(f'Starting get_request_flaresolver for {url} with timeout={timeout}, retries={retries}, time_between_retries={time_between_retries}')
-    for attempt in range(retries):
-        logger.debug(f'Attempt {attempt + 1} for {url} using FlareSolver')
-        try:
-            response = requests.post(
-                flaresolver_url,
-                headers=FLARE_HEADERS,
-                json={
-                    'cmd': 'request.get',
-                    'url': url,
-                    'maxTimeout': timeout * 1000
-                },
-                timeout=timeout
-            )
-            response.raise_for_status()
-            logger.debug(f'Successful response for {url} on attempt {attempt + 1} using FlareSolver')
-            return response
-
-        except requests.exceptions.ConnectionError as e:
-            logger.error(f'Connection error ({attempt + 1}/{retries}), check FlareSolver host: {flaresolver_url}: {e}')
-        except requests.exceptions.Timeout as e:
-            logger.error(f'Request timed out ({attempt + 1}/{retries}): {e}')
-        except requests.exceptions.InvalidSchema as e:
-            logger.error(f'Invalid FlareSolver URL schema "{flaresolver_url}": {e}')
-            break  # Don't retry on invalid schema
-        except requests.exceptions.HTTPError as e:
-            logger.error(f'HTTP error ({attempt + 1}/{retries}): {e}')
-        except requests.exceptions.RequestException as e:
-            logger.error(f'Request failed ({attempt + 1}/{retries}): {e}')
-        except json.JSONDecodeError as e:
-            logger.error(f'Invalid JSON response ({attempt + 1}/{retries}): {e}')
-
-        if attempt < retries - 1:
-            logger.debug(f'Waiting {time_between_retries} seconds before retrying')
-            time.sleep(time_between_retries)  # Wait before retrying
-    logger.debug(f'Failed to get a successful response for {url} using FlareSolver after {retries} attempts')
-    return None
-
-
-def get_html_content(url: str,
-                     retries: int = 5,
-                     flaresolver: bool = True,
-                     flaresolver_url: str = FLARESOLVER_URL,
-                     time_between_retries: int = 1,
-                     force_flaresolver: bool = FORCE_FLARESOLVER) -> str | None:
-    logger.debug(f'Starting get_html_content for {url} with retries={retries}, flaresolver={flaresolver}, flaresolver_url={flaresolver_url}, time_between_retries={time_between_retries}, force_flaresolver={force_flaresolver}')
-    # First try with common HTTP request
-    if not force_flaresolver:
-        response = get_request(
-            url, timeout=20, retries=retries, time_between_retries=time_between_retries)
-        if not response:
-            logger.warning(f'Failed to get response from {url} using common HTTP request')
-        elif not response.ok:
-            logger.warning(f'Response with errors from {url} using common HTTP request')
-        else:
-            logger.debug(f'Successfully retrieved HTML content from {url} using common HTTP request')
-            return response.text
-
-    # If flaresolver is disabled, return None
-    if not flaresolver:
-        logger.debug(f'Flaresolver is disabled, returning None for {url}')
-        return None
-
-    # Try with Flaresolver
-    logger.debug(f'Trying with Flaresolver for {url}')
-    response = get_request_flaresolver(
-        url, timeout=20, flaresolver_url=flaresolver_url, time_between_retries=time_between_retries)
-    if not response:
-        logger.critical(f'Failed to get response from {url} using FlareSolver')
-        return None
-    if not response.ok:
-        logger.critical(f'Response with errors from {url} using FlareSolver')
-        return None
-
-    response_json = response.json()
-    if 'solution' not in response_json:
-        logger.critical(f'No solution found in FlareSolver response for {url}')
-        return None
-    if 'response' not in response_json['solution']:
-        logger.critical(f'No response found in FlareSolver solution for {url}')
-        return None
-    logger.debug(f'Successfully retrieved HTML content from {url} using FlareSolver')
-    return response_json['solution']['response']

web_novel_scraper-1.1.0/web_novel_scraper/version.py

@@ -1 +0,0 @@
-__version__ = "1.1.0"