web-novel-scraper 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

web_novel_scraper/__main__.py

@@ -52,7 +52,7 @@ def validate_date(ctx, param, value):
 
 # COMMON ARGUMENTS
 title_option = click.option(
-    '-t', '--title', type=str, required=True, help='Title of the novel, this server as the identifier.')
+    '-t', '--title', type=str, required=True, envvar='SCRAPER_NOVEL_TITLE', help='Title of the novel, this server as the identifier.')
 novel_base_dir_option = click.option(
     '-nb', '--novel-base-dir', type=str, help='Alternative base directory for the novel files.')
 
@@ -330,19 +330,25 @@ def show_toc(title, novel_base_dir):
 @click.option('--update-html', is_flag=True, default=False, show_default=True, help='If the chapter HTML is saved, it will be updated.')
 def scrap_chapter(title, novel_base_dir, chapter_url, chapter_num, update_html):
     """Scrap a chapter of a novel."""
+    if (chapter_url is None and chapter_num is None) or (chapter_url and chapter_num):
+        raise click.UsageError("You must set exactly one: --chapter-url o --chapter-num.")
+
     novel = obtain_novel(title, novel_base_dir)
-    if not chapter_url and not chapter_num:
-        click.echo('Chapter URL or chapter number should be set.', err=True)
-    if chapter_num and chapter_url:
-        click.echo('It should be either chapter URL or chapter number.', err=True)
-    if chapter_num <= 0 or chapter_num > len(novel.chapters):
-        raise click.BadParameter(
-            'Chapter number should be positive and an existing chapter.', param_hint='--chapter-num')
-    chapter = novel.scrap_chapter(
-        chapter_url=chapter_url, chapter_idx=chapter_num - 1, update_html=update_html)
+
+    if chapter_num is not None:
+        if chapter_num <= 0 or chapter_num > len(novel.chapters):
+            raise click.BadParameter(
+                'Chapter number should be positive and an existing chapter.', param_hint='--chapter-num')
+        chapter = novel.scrap_chapter(chapter_idx=chapter_num - 1,
+                                      update_html=update_html)
+
+    else:
+        chapter = novel.scrap_chapter(chapter_url=chapter_url,
+                                      update_html=update_html)
+
     if not chapter:
-        click.echo('Chapter number or URL not found.', err=True)
-        return
+        raise click.ClickException('Chapter not found or scrap failed.')
+
     click.echo(chapter)
     click.echo('Content:')
     click.echo(chapter.chapter_content)

web_novel_scraper/custom_processor/__init__.py

@@ -0,0 +1,2 @@
+from .custom_processor import CustomProcessor, ProcessorRegistry
+from .sites import royalroad, genesis

web_novel_scraper/custom_processor/custom_processor.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+class CustomProcessor(ABC):
+    @abstractmethod
+    def process(self, html: str) -> Any:
+        """Process the HTML content using custom logic"""
+        pass
+
+class ProcessorRegistry:
+    _processors: Dict[str, Dict[str, CustomProcessor]] = {}
+
+    @classmethod
+    def register(cls, host: str, content_type: str, processor: CustomProcessor):
+        if host not in cls._processors:
+            cls._processors[host] = {}
+        cls._processors[host][content_type] = processor
+
+    @classmethod
+    def get_processor(cls, host: str, content_type: str) -> CustomProcessor:
+        return cls._processors.get(host, {}).get(content_type)
+
+    @classmethod
+    def has_processor(cls, host: str, content_type: str) -> bool:
+        return bool(cls.get_processor(host, content_type))

web_novel_scraper/custom_processor/sites/genesis.py

@@ -0,0 +1,46 @@
+import re
+import json
+from typing import List, Optional
+from ..custom_processor import CustomProcessor, ProcessorRegistry
+
+GENESIS_STUDIO_VIEWER_URL = 'https://genesistudio.com/viewer'
+
+class GenesisChaptersProcessor(CustomProcessor):
+    def process(self, html: str) -> Optional[List[dict]]:
+        pattern = r',chapters:\s*{\s*free:\s*(\[.*?"}}])'
+        match = re.search(pattern, html, re.DOTALL)
+
+        if not match:
+            if not match:
+                return None
+
+        try:
+            chapters_json = match.group(1).strip()
+            replaces = {
+                "chapter_title:": '"chapter_tile":',
+                "id:": '"id":',
+                "nsfw:": '"nsfw":',
+                "required_tier:": '"required_tier":',
+                "date_created:": '"date_created":',
+                "spoiler_title:": '"spoiler_title":',
+                "chapter_number:": '"chapter_number":',
+                "novel:": '"novel":',
+            }
+            # Ensure the JSON string ends properly
+            if not chapters_json.endswith(']'):
+                chapters_json += ']'
+            for old_key, new_key in replaces.items():
+                chapters_json = chapters_json.replace(old_key, new_key)
+            # print(f"Extracted JSON: {chapters_json[12200:12300]}")  # Debug print
+            chapters = json.loads(chapters_json)
+            chapters_url = []
+            for chapter in chapters:
+                chapters_url.append(f"{GENESIS_STUDIO_VIEWER_URL}/{chapter['id']}")
+            print(chapters)
+            return chapters_url
+            
+        except (json.JSONDecodeError, IndexError) as e:
+            print(f"Error processing JSON: {str(e)}")
+            return None
+
+ProcessorRegistry.register('genesistudio.com', 'index', GenesisChaptersProcessor())

web_novel_scraper/custom_processor/sites/royalroad.py

@@ -0,0 +1,22 @@
+import re
+import json
+from typing import List, Optional
+from ..custom_processor import CustomProcessor, ProcessorRegistry
+
+class RoyalRoadChaptersProcessor(CustomProcessor):
+    def process(self, html: str) -> Optional[List[dict]]:
+        pattern = r'window\.chapters\s*=\s*(\[.*?\]);'
+        match = re.search(pattern, html, re.DOTALL)
+
+        if not match:
+            return None
+
+        try:
+            chapters_json = match.group(1)
+            chapters = json.loads(chapters_json)
+            chapters = [chapter['url'] for chapter in chapters if 'url' in chapter]
+            return chapters
+        except (json.JSONDecodeError, IndexError):
+            return None
+
+ProcessorRegistry.register('www.royalroad.com', 'index', RoyalRoadChaptersProcessor())

web_novel_scraper/decode.py

@@ -1,8 +1,10 @@
 import os
 import json
 from pathlib import Path
+from typing import Optional
 
 from . import logger_manager
+from .custom_processor.custom_processor import ProcessorRegistry
 
 from bs4 import BeautifulSoup
 
@@ -41,17 +43,92 @@ class Decoder:
         self.decode_guide = self._get_element_by_key(
             DECODE_GUIDE, 'host', host)
 
-    def decode_html(self, html: str, content_type: str):
+    def get_chapter_urls(self, html: str) -> list[str]:
+        logger.debug('Obtaining chapter URLs...')
+        chapter_urls = self.decode_html(html, 'index')
+
+        if chapter_urls is None:
+            logger.critical(f"Failed to obtain chapter URLs for {self.host}")
+            raise ValueError(f"Failed to obtain chapter URLs for {self.host}")
+
+        if isinstance(chapter_urls, str):
+            logger.warning('When obtaining chapter urls, obtained a String but expected a List')
+            logger.warning('Check decode config')
+            chapter_urls = [chapter_urls]
+
+        return chapter_urls
+
+    def get_toc_next_page_url(self, html: str) -> Optional[str]:
+        logger.debug('Obtaining toc next page URL...')
+        toc_next_page_url = self.decode_html(html, 'next_page')
+        if toc_next_page_url is None:
+            logger.debug('No next page URL found, assuming last page...')
+            return None
+        return toc_next_page_url
+
+    def get_chapter_title(self, html: str) -> Optional[str]:
+        logger.debug('Obtaining chapter title...')
+        chapter_title = self.decode_html(html, 'title')
+        if chapter_title is None:
+            logger.debug(f'No chapter_title found.')
+        return chapter_title
+
+    def get_chapter_content(self, html: str, save_title_to_content: bool, chapter_title: str) -> str:
+        logger.debug('Obtaining chapter content...')
+        full_chapter_content = ""
+        chapter_content = self.decode_html(html, 'content')
+
+        if chapter_content is None:
+            logger.critical('No content found on chapter')
+            raise ValueError('No content found on chapter')
+
+        if save_title_to_content:
+            logger.debug('Saving chapter title to content...')
+            full_chapter_content += f'<h4>{chapter_title}</h4>'
+
+        if isinstance(chapter_content, list):
+            logger.debug(f'{len(chapter_content)} paragraphs found in chapter')
+            logger.debug('Converting list of paragraphs to a single string')
+            for paragraph in chapter_content:
+                full_chapter_content += str(paragraph)
+        else:
+            logger.debug('Chapter content is not a list, no conversion made')
+            full_chapter_content += str(chapter_content)
+        return full_chapter_content
+
+    def decode_html(self, html: str, content_type: str) -> str | list[str] | None:
+        logger.debug(f'Decoding HTML...')
+        logger.debug(f'Content type: {content_type}')
+        logger.debug(f'Decode guide: {DECODE_GUIDE_FILE}')
+        logger.debug(f'Host: {self.host}')
         if not content_type in self.decode_guide:
-            logger.error(f'{content_type} key does not exists on decode guide {
-                         DECODE_GUIDE_FILE} for host {self.host}')
-            return
-        soup = BeautifulSoup(html, 'html.parser')
+            logger.critical(f'{content_type} key does not exists on decode guide {DECODE_GUIDE_FILE}'
+                            f'for host {self.host}')
+            raise ValueError(f'{content_type} key does not exists on decode guide {DECODE_GUIDE_FILE}'
+                            f'for host {self.host}')
+
+        if ProcessorRegistry.has_processor(self.host, content_type):
+            logger.debug(f'Host {self.host} will use a custom processor')
+            processor = ProcessorRegistry.get_processor(self.host, content_type)
+            return processor.process(html)
+
+        logger.debug('Starting HTML parsing...')
+        try:
+            soup = BeautifulSoup(html, 'html.parser')
+        except Exception as e:
+            logger.critical(f'Error parsing HTML with BeautifulSoup: {e}')
+            raise ValueError(f'Error parsing HTML with BeautifulSoup: {e}')
+
         decoder = self.decode_guide[content_type]
         elements = self._find_elements(soup, decoder)
         if not elements:
-            logger.warning(f'{content_type} not found on html using {
-                           DECODE_GUIDE_FILE} for host {self.host}')
+            logger.warning(f'{content_type} not found on html using {DECODE_GUIDE_FILE} '
+                           f'for host {self.host}')
+
+        # Investigate this conditional
+        if content_type == 'title' and isinstance(elements, list):
+            logger.debug('Joining titles...')
+            return ' '.join(elements)
         return elements
 
     def has_pagination(self, host: str = None):
@@ -81,8 +158,11 @@ class Decoder:
 
         return "\n".join([line.strip() for line in str(soup).splitlines() if line.strip()])
 
-    def _find_elements(self, soup: BeautifulSoup, decoder: dict):
+    @staticmethod
+    def _find_elements(soup: BeautifulSoup, decoder: dict):
+        logger.debug('Finding elements...')
         selector = decoder.get('selector')
+        elements = []
         if selector is None:
             selector = ''
             element = decoder.get('element')
@@ -91,32 +171,46 @@ class Decoder:
             attributes = decoder.get('attributes')
 
             if element:
+                logger.debug(f'Using element "{element}"')
                 selector += element
             if _id:
+                logger.debug(f'Using id "{_id}"')
                 selector += f'#{_id}'
             if _class:
+                logger.debug(f'Using class "{_class}"')
                 selector += f'.{_class}'
             if attributes:
                 for attr, value in attributes.items():
-                    selector += f'[{attr}="{value}"]' if value else f'[{attr}]'
+                    logger.debug(f'Using attribute "{attr}"')
+                    if value is not None:
+                        logger.debug(f'With value "{value}"')
+                        selector += f'[{attr}="{value}"]'
+                    else:
+                        selector += f'[{attr}]'
             selectors = [selector]
         else:
+            logger.debug(f'Using selector "{selector}"')
             if XOR_SEPARATOR in selector:
+                logger.debug(f'Found XOR_OPERATOR "{XOR_SEPARATOR}" in selector')
+                logger.debug('Splitting selectors...')
                 selectors = selector.split(XOR_SEPARATOR)
             else:
                 selectors = [selector]
 
         for selector in selectors:
-            logger.debug(f'Attempt using selector {selector}')
+            logger.debug(f'Searching using selector "{selector}"...')
             elements = soup.select(selector)
             if elements:
-                logger.debug(f'{len(elements)} found using selector {selector}')
+                logger.debug(f'{len(elements)} found using selector "{selector}"')
                 break
+            logger.debug(f'No elements found using selector "{selector}"')
 
         extract = decoder.get('extract')
         if extract:
+            logger.debug(f'Extracting from elements...')
             if extract["type"] == "attr":
                 attr_key = extract["key"]
+                logger.debug(f'Extracting value from attribute "{attr_key}"...')
                 elements_aux = elements
                 elements = []
                 for element in elements_aux:
@@ -125,15 +219,34 @@ class Decoder:
                         if attr:
                             elements.append(attr)
                     except KeyError:
+                        logger.debug(f'Attribute "{attr_key}" not found')
+                        logger.debug('Ignoring...')
                         pass
+                logger.debug(f'{len(elements)} elements found using attribute "{attr_key}"')
             if extract["type"] == "text":
+                logger.debug('Extracting text from elements...')
                 elements = [element.string for element in elements]
+
+        if not elements:
+            logger.error('No elements found, returning "None"')
+            return None
+
         inverted = decoder.get('inverted')
         if inverted:
+            logger.debug('Inverted option activate')
+            logger.debug('Inverting elements order...')
             elements = elements[::-1]
-        return elements if decoder.get('array') else elements[0] if elements else None
 
-    def _get_element_by_key(self, json_data, key, value):
+        if decoder.get('array'):
+            logger.debug('Array option activated')
+            logger.debug('Returning elements a list')
+            return elements
+        logger.debug('Array option not activated')
+        logger.debug('Returning only first element...')
+        return elements[0]
+
+    @staticmethod
+    def _get_element_by_key(json_data, key, value):
         for item in json_data:
             if item[key] == value:
                 return item

web_novel_scraper/decode_guide/decode_guide.json

@@ -24,7 +24,11 @@
             "class": null,
             "selector": null,
             "attributes": null,
-            "array": true
+            "array": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
         },
         "next_page": {
             "element": "p",
@@ -60,7 +64,11 @@
             "class": null,
             "selector": "div.m-newest2 ul li a",
             "attributes": null,
-            "array": true
+            "array": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
         },
         "next_page": {
             "element": null,
@@ -72,7 +80,7 @@
         }
     },
     {
-        "host": "royalroad.com",
+        "host": "www.royalroad.com",
         "has_pagination": false,
         "title": {
             "element": null,
@@ -95,12 +103,7 @@
             "array": true
         },
         "index": {
-            "element": null,
-            "id": null,
-            "class": null,
-            "selector": "tr.chapter-row td a",
-            "attributes": null,
-            "array": true
+            "use_custom_processor": true
         },
         "next_page": {
             "element": null,
@@ -140,7 +143,11 @@
             "class": null,
             "selector": "ul.list-chapter li a",
             "attributes": null,
-            "array": true
+            "array": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
         },
         "next_page": {
             "element": null,
@@ -167,10 +174,10 @@
             }
         },
         "content": {
-            "element": "div#chr-content",
+            "element": null,
             "id": null,
             "class": null,
-            "selector": null,
+            "selector": "div#chr-content p",
             "attributes": null,
             "array": true
         },
@@ -180,7 +187,11 @@
             "class": null,
             "selector": "ul.list-chapter li a",
             "attributes": null,
-            "array": true
+            "array": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
         },
         "next_page": {
             "element": null,
@@ -207,7 +218,82 @@
         "index": {
             "element": "ul.main li a",
             "array": true,
-            "inverted": true
+            "inverted": true,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
+        }
+    },
+    {
+        "host": "genesistudio.com",
+        "has_pagination": false,
+        "title": {
+            "element": null,
+            "id": null,
+            "class": null,
+            "selector": "p.leading-none span",
+            "attributes": null,
+            "array": true,
+            "extract": {
+                "type": "text",
+                "key": "text"
+            }
+        },
+        "content": {
+            "element": "p",
+            "id": null,
+            "class": "narration",
+            "selector": null,
+            "attributes": null,
+            "array": true,
+            "extract": {
+                "type": "text",
+                "key": "text"
+            }
+        },
+        "index": {
+            "use_custom_processor": true
+        },
+        "next_page": {
+            "element": null,
+            "id": null,
+            "class": null,
+            "selector": null,
+            "attributes": null,
+            "array": true
+        }
+    },
+    {
+        "host": "hostednovel.com",
+        "has_pagination": true,
+        "title": {
+            "selector": "span#chapter-title",
+            "extract": {
+                "type": "text"
+            }
+        },
+        "content": {
+            "element": "div",
+            "id": "chapter-content",
+            "array": true
+        },
+        "index": {
+            "selector": "li ul li.flow-root a",
+            "array": true,
+            "inverted": false,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
+        },
+        "next_page": {
+            "selector": "a:has(span:contains('Next'))",
+            "array": false,
+            "extract": {
+                "type": "attr",
+                "key": "href"
+            }
         }
     }
 ]

web_novel_scraper/file_manager.py

@@ -7,6 +7,7 @@ from pathlib import Path
 import shutil
 from dotenv import load_dotenv
 from ebooklib import epub
+import unicodedata
 
 from . import logger_manager
 
@@ -77,6 +78,16 @@ class FileManager:
     def save_chapter_html(self, filename: str, content: str):
         full_path = self.novel_chapters_dir / filename
         logger.debug(f'Saving chapter to {full_path}')
+        content = unicodedata.normalize('NFKC', content)
+        char_replacements = {
+            "â": "'",    # Reemplazar â con apóstrofe
+            "\u2018": "'", # Comillda simple izquierda Unicode
+            "\u2019": "'", # Comilla simple derecha Unicode
+            "\u201C": '"', # Comilla doble izquierda Unicode
+            "\u201D": '"', # Comilla doble derecha Unicode
+        }
+        for old_char, new_char in char_replacements.items():
+            content = content.replace(old_char, new_char)
         _save_content_to_file(full_path, content)
 
     def load_chapter_html(self, filename: str):

web_novel_scraper/novel_scraper.py

@@ -220,6 +220,7 @@ class Novel:
             self.decoder = Decoder(self.host)
         elif update_host:
             self.decoder = Decoder(utils.obtain_host(self.toc_main_url))
+        self.save_novel()
 
     def add_toc_html(self, html: str, host: str = None) -> None:
         if self.toc_main_url:
@@ -248,7 +249,7 @@ class Novel:
         toc_not_exists = not all_tocs_content and self.toc_main_url is None
         if toc_not_exists:
             logger.critical(
-                'There is no toc html and no toc url setted, unable to get toc.')
+                'There is no toc html and no toc url set, unable to get toc.')
             return False
 
         reload_files = reload_files and self.toc_main_url is not None
@@ -259,18 +260,16 @@ class Novel:
             toc_content = self._add_toc(self.toc_main_url)
             all_tocs_content.append(toc_content)
             if self.decoder.has_pagination():
-                next_page = self._get_next_page_from_toc_content(toc_content)
+                next_page = self.decoder.get_toc_next_page_url(toc_content)
                 while next_page:
                     toc_content = self._add_toc(next_page)
-                    next_page = self._get_next_page_from_toc_content(
-                        toc_content)
+                    next_page = self.decoder.get_toc_next_page_url(toc_content)
                     all_tocs_content.append(toc_content)
 
         # Now we get the links from the toc content
         self.chapters_url_list = []
         for toc_content in all_tocs_content:
-            chapters_url_from_toc_content = self._get_chapter_urls_from_toc_content(
-                toc_content)
+            chapters_url_from_toc_content = self.decoder.get_chapter_urls(toc_content)
             if chapters_url_from_toc_content is None:
                 logger.error('Chapters url not found on toc_content')
                 return False
@@ -307,35 +306,39 @@ class Novel:
         return chapter_list
 
     def scrap_chapter(self, chapter_url: str = None, chapter_idx: int = None, update_html: bool = False) -> Chapter:
+        logger.info('Scraping Chapter...')
+        chapter = None
         if not utils.check_exclusive_params(chapter_url, chapter_idx):
-            logger.error(
-                'chapter_url and chapter_id, only one needs to be setted')
-            return
+            raise ValueError("chapter_url and chapter_id, only one needs to be set")
 
         if chapter_url is not None:
+            logger.debug(f'Using chapter url: {chapter_url}')
             chapter = self._get_chapter_by_url(chapter_url=chapter_url)
             if chapter is None:
+                logger.warning(f'Chapter with url "{chapter_url}" does not exist, generating one...')
                 chapter = Chapter(chapter_url=chapter_url)
 
         if chapter_idx is not None:
+            logger.debug(f'Using chapter index: {chapter_idx}')
             if chapter_idx < 0 or chapter_idx >= len(self.chapters):
-                logger.error(f'Could not find chapter with idx {chapter_idx}')
-                return
-            chapter = self.chapters[chapter_idx]
+                logger.critical(f'Could not find chapter with idx {chapter_idx}')
+                raise ValueError(f'Could not find chapter with idx {chapter_idx}')
 
+            chapter = self.chapters[chapter_idx]
+        if update_html:
+            logger.debug('HTML will be updated...')
         chapter = self._get_chapter(chapter,
                                     reload=update_html)
 
         if not chapter.chapter_html or not chapter.chapter_html_filename:
-            logger.warning(f'Failed to create chapter on link: "{
-                           chapter_url}" on path "{chapter.chapter_html_filename}"')
-            return
+            logger.critical(f'Failed to create chapter on link: "{chapter_url}" '
+                           f'on path "{chapter.chapter_html_filename}"')
+            raise ValueError(f'Failed to create chapter on link: "{chapter_url}" '
+                           f'on path "{chapter.chapter_html_filename}"')
 
-        # We get the title and content, if there's no title, we autogenerate one.
+        # We get the chapter title and content
+        # We pass an index so we can autogenerate a Title
         chapter = self._decode_chapter(chapter=chapter, idx_for_chapter_name=chapter_idx)
-        if not chapter.chapter_content:
-            logger.error('Content not found')
-            return
 
         logger.info(f'Chapter scrapped from link: {chapter_url}')
         return chapter
@@ -506,22 +509,6 @@ class Novel:
         self.file_manager.add_toc(content)
         return content
 
-    def _get_chapter_urls_from_toc_content(self, toc_content: str) -> list[str]:
-        toc_elements = self.decoder.decode_html(toc_content, 'index')
-        try:
-            toc_urls = [toc_element['href'] for toc_element in toc_elements]
-        except KeyError as e:
-            logger.error(f'{e} not found on the Tag elements decoded from TOC')
-            return
-        if toc_urls:
-            return toc_urls
-        logger.warning('No chapter links found on toc content')
-
-    def _get_next_page_from_toc_content(self, toc_content: str) -> str:
-        next_page = self.decoder.decode_html(toc_content, 'next_page')
-        if next_page:
-            return next_page[0]['href']
-
     def _add_or_update_chapter_data(self, chapter: Chapter, link_idx: int = None, save_in_file: bool = True) -> None:
         if link_idx:
             chapter_idx = link_idx
@@ -579,35 +566,28 @@ class Novel:
         self.save_novel()
 
     def _decode_chapter(self, chapter: Chapter, idx_for_chapter_name: str = None) -> Chapter:
-        chapter_title = None
-
+        logger.debug('Decoding chapter...')
         if chapter.chapter_html is None:
+            logger.debug(f'No HTML content found, requesting HTML content...')
             chapter = self._get_chapter(chapter)
 
             if not chapter.chapter_html:
-                logger.error(f'No chapter content found for chapter link {
-                             chapter.chapter_url} on file {chapter.chapter_html_filename}')
-                return None
-
-        paragraphs = self.decoder.decode_html(chapter.chapter_html, 'content')
-
-        if not paragraphs:
-            if chapter:
-                logger.warning(f'No paragraphs found in chapter link {
-                    chapter.chapter_url} on file {chapter.chapter_html_filename}')
+                raise ValueError(f'Chapter HTML could not be obtained for chapter link "{chapter.chapter_url}" '
+                                 f'on file "{chapter.chapter_html_filename}"')
 
-        chapter_title = self.decoder.decode_html(chapter.chapter_html, 'title')
+        logger.debug('Obtaining chapter title...')
+        chapter_title = self.decoder.get_chapter_title(chapter.chapter_html)
         if not chapter_title:
-            chapter_title = f'{self.metadata.novel_title} Chapter {
-                idx_for_chapter_name}'
+            logger.debug('No chapter title found, generating one...')
+            chapter_title = f'{self.metadata.novel_title} Chapter {idx_for_chapter_name}'
         chapter.chapter_title = str(chapter_title)
+        logger.debug(f'Chapter title: "{chapter_title}"')
 
-        chapter.chapter_content = ""
-        if self.scraper_behavior.save_title_to_content:
-            chapter.chapter_content += f'<h4>{chapter_title}</h4>'
-        logger.info(f'{len(paragraphs)} paragraphs found in chapter')
-        for paragraph in paragraphs:
-            chapter.chapter_content += str(paragraph)
+        logger.debug('Obtaining chapter content...')
+        chapter.chapter_content = self.decoder.get_chapter_content(chapter.chapter_html,
+                                                           self.scraper_behavior.save_title_to_content,
+                                                           chapter.chapter_title)
+        logger.debug('Chapter successfully decoded')
 
         return chapter
 
@@ -631,7 +611,7 @@ class Novel:
         if self.metadata.start_date:
             date_metadata += self.metadata.start_date
         # Calibre specification doesn't use end_date.
-        # For now we use a custom metadata
+        # For now, we use a custom metadata
         # https://idpf.org/epub/31/spec/epub-packages.html#sec-opf-dcdate
         # if self.metadata.end_date:
         #     date_metadata += f'/{self.metadata.end_date}'

web_novel_scraper/request_manager.py

@@ -13,7 +13,6 @@ FORCE_FLARESOLVER = os.getenv('FORCE_FLARESOLVER', '0') == '1'
 
 logger = logger_manager.create_logger('GET HTML CONTENT')
 
-
 def get_request(url: str,
                 timeout: int = 20,
                 retries: int = 3,

web_novel_scraper/version.py

@@ -1 +1 @@
-__version__ = "1.0.2"
+__version__ = "1.0.4"

{web_novel_scraper-1.0.2.dist-info → web_novel_scraper-1.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-novel-scraper
-Version: 1.0.2
+Version: 1.0.4
 Summary: Python tool that allows you to scrape web novels from various sources and save them to more readable formats like EPUB.
 Project-URL: Homepage, https://github.com/ImagineBrkr/web-novel-scraper
 Project-URL: Documentation, https://web-novel-scraper.readthedocs.io
@@ -138,7 +138,7 @@ In the example above:
 The following commands are available in the Web Novel Scraping CLI:
 
 ```bash
-Usage: main.py [OPTIONS] COMMAND [ARGS]...
+Usage: web-novel-scraper [OPTIONS] COMMAND [ARGS]...
 
   CLI Tool for web novel scraping.
 
@@ -175,32 +175,32 @@ Here are some basic examples:
 
 ### Example 1: Creating a Novel using a main URL
 ```bash
-python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-main-url 'https://page.me/Novel-1/toc' --cover 'cover.jpg'
+web-novel-scraper create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-main-url 'https://page.me/Novel-1/toc' --cover 'cover.jpg'
 ```
 Some pages have too much JavaScript, so you can just copy the HTML manually to a file and create the novel from it:
 ```bash
-python src/main.py create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-html 'toc.html' --host 'page.me' --cover 'cover.jpg'
+web-novel-scraper create-novel --title 'Novel 1' --author 'ImagineBrkr' --toc-html 'toc.html' --host 'page.me' --cover 'cover.jpg'
 ```
 If there is more than one page for the TOC, you can add them:
 ```bash
-python src/main.py add-toc-html --title 'Novel 1' --toc-html 'toc2.html'
+web-novel-scraper add-toc-html --title 'Novel 1' --toc-html 'toc2.html'
 ```
 You can create the chapters from this TOC, or synchronize if they were already created but there are new chapters.
 ```bash
-python src/main.py sync-toc --title 'Novel 1'
+web-novel-scraper sync-toc --title 'Novel 1'
 ```
 The default directory will be %APPDATA%/ImagineBrkr/web-novel-scraper for Windows, all the files will be saved there, but you can change it.
 
 ### Example 2: Requesting files
 We can now download all the chapters
 ```bash
-python src/main.py request-all-chapters --title 'Novel 1'
+web-novel-scraper request-all-chapters --title 'Novel 1'
 ```
 
 ### Example 3: Saving to EPUB
 With 
 ```bash
-python src/main.py save-novel-to-epub --title 'Novel 1'
+web-novel-scraper save-novel-to-epub --title 'Novel 1'
 ```
 
 For more detailed usage and options, use --help for each command.

web_novel_scraper-1.0.4.dist-info/RECORD

@@ -0,0 +1,18 @@
+web_novel_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+web_novel_scraper/__main__.py,sha256=OQQVX5CttmAkUwdrnjBSjKPaoh_boUI2ysHi3rLGOSs,17769
+web_novel_scraper/decode.py,sha256=0RMHx1buR01KhuXiVQwdSpCGN960Xh-iPw1eYHxLeDg,10181
+web_novel_scraper/file_manager.py,sha256=Q3DH-c8fWz9sziMps7A3p_sQoDMEpqBket07Agh-__Q,11898
+web_novel_scraper/logger_manager.py,sha256=A-a4bhYI4YCEuSJd9E3WH_kanJ7YCASMwheBzObZK4Q,1972
+web_novel_scraper/novel_scraper.py,sha256=Notk0O94HZrO-MVKDGCBL0VopApFchn13FO2_N3ZfRM,28418
+web_novel_scraper/request_manager.py,sha256=VtGpRi5b_Dp3h8viCdt7yMCb9M21Lk7oLC_Q_0EkXH8,6448
+web_novel_scraper/utils.py,sha256=vq5ROuPv04k3MhbksTe0ni_yP0i_a7T_33mkBB1DUbQ,2076
+web_novel_scraper/version.py,sha256=acuR_XSJzp4OrQ5T8-Ac5gYe48mUwObuwjRmisFmZ7k,22
+web_novel_scraper/custom_processor/__init__.py,sha256=iy4tjivMjshSzc52--aa-jK53qu9VwdK-6p4vuQc6oc,103
+web_novel_scraper/custom_processor/custom_processor.py,sha256=h1MPl6JU_C2Mc7SqK70LsNQHpDzSL6QyraMIQ87HcMM,870
+web_novel_scraper/custom_processor/sites/genesis.py,sha256=xV0eybI0ieHR5gn4yWXI74l99Eayhqs16PIYs-BrPjE,1843
+web_novel_scraper/custom_processor/sites/royalroad.py,sha256=_2PsFC_w3RJCUkAPoRn-7R2jlzl3XsG4WYtRaQkp6lg,787
+web_novel_scraper/decode_guide/decode_guide.json,sha256=IBBzbSSVO-yQ5PCY7o8ralnaonMwBpEZW1v1TStiVqc,7582
+web_novel_scraper-1.0.4.dist-info/METADATA,sha256=IhvDqK_Gz1POjzbH2cQVUYql1dhZJvdHnM9R--le0uc,8423
+web_novel_scraper-1.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+web_novel_scraper-1.0.4.dist-info/entry_points.txt,sha256=bqRvStfvSprSJc2EJXgKIbggWOXSePHFfVIZWy_plDQ,69
+web_novel_scraper-1.0.4.dist-info/RECORD,,

web_novel_scraper-1.0.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-web_novel_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-web_novel_scraper/__main__.py,sha256=PBIAG1vshnRdvYwyeD-RxlHS5mNEby-d4puV9kEyfpA,17615
-web_novel_scraper/decode.py,sha256=dqGv_8nFSKwO6GBj3jhaO9SQeLHeBjDzoV1C_YcN40k,5085
-web_novel_scraper/file_manager.py,sha256=PJu8kKeng49DTNQBbbMekFtIcTZOkeCEjFYqYJarv9M,11363
-web_novel_scraper/logger_manager.py,sha256=A-a4bhYI4YCEuSJd9E3WH_kanJ7YCASMwheBzObZK4Q,1972
-web_novel_scraper/novel_scraper.py,sha256=eiic2i3AdK9lcFK9aNb4d8ptnKv9ua1B_9kcUY8_liM,28660
-web_novel_scraper/request_manager.py,sha256=0M_ekBuDCMRGZIWxDbZ_yAwPOxJr2mBpP-Yj8zsE13o,6449
-web_novel_scraper/utils.py,sha256=vq5ROuPv04k3MhbksTe0ni_yP0i_a7T_33mkBB1DUbQ,2076
-web_novel_scraper/version.py,sha256=Y3LSfRioSl2xch70pq_ULlvyECXyEtN3krVaWeGyaxk,22
-web_novel_scraper/decode_guide/decode_guide.json,sha256=Q4v-OZh_1MwdrFxDDVvj8T3evW3zzbSapRaGwFCdnX8,5425
-web_novel_scraper-1.0.2.dist-info/METADATA,sha256=OBhkSUWS02JIFh4qbsRdS_s_UL15_gAOWAqsQu-Day4,8419
-web_novel_scraper-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-web_novel_scraper-1.0.2.dist-info/entry_points.txt,sha256=bqRvStfvSprSJc2EJXgKIbggWOXSePHFfVIZWy_plDQ,69
-web_novel_scraper-1.0.2.dist-info/RECORD,,