web-novel-scraper 1.1.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

web_novel_scraper/novel_scraper.py

@@ -11,6 +11,7 @@ from .file_manager import FileManager
 from . import utils
 
 from .request_manager import get_html_content
+from .config_manager import ScraperConfig
 
 logger = logger_manager.create_logger('NOVEL SCRAPPING')
 
@@ -18,7 +19,6 @@ logger = logger_manager.create_logger('NOVEL SCRAPPING')
 @dataclass_json
 @dataclass
 class Metadata:
-    novel_title: str
     author: Optional[str] = None
     start_date: Optional[str] = None
     end_date: Optional[str] = None
@@ -105,10 +105,11 @@ class Chapter:
         return self.chapter_title < another.chapter_title
 
 
-@dataclass_json
+@dataclass_json(undefined=Undefined.EXCLUDE)
 @dataclass
 class Novel:
-    metadata: Metadata
+    metadata: Metadata = None
+    title: str = None
     scraper_behavior: ScraperBehavior = None
     chapters: list[Chapter] = field(default_factory=list)
     toc_main_url: Optional[str] = None
@@ -116,30 +117,23 @@ class Novel:
     host: str = None
 
     def __init__(self,
-                 novel_title: str = None,
+                 title: str,
                  toc_main_url: str = None,
                  toc_html: str = None,
                  chapters_url_list: list[str] = None,
                  metadata: Metadata = None,
                  chapters: list[Chapter] = None,
-                 novel_base_dir: str = None,
                  scraper_behavior: ScraperBehavior = None,
-                 host: str = None):
-
+                 host: str = None
+                 ):
         if toc_main_url and toc_html:
-            logger.error('There can only be one or toc_main_url or toc_html')
-            sys.exit(1)
+            logger.critical('There can only be one or toc_main_url or toc_html')
+            raise ValueError('There can only be one or toc_main_url or toc_html')
 
+        self.title = title
+        self.metadata = Metadata()
         if metadata is not None:
             self.metadata = metadata
-        elif novel_title is not None:
-            self.metadata = Metadata(novel_title)
-        else:
-            logger.error('You need to set "novel_title" or "metadata".')
-            sys.exit(1)
-
-        self.file_manager = FileManager(novel_title=self.metadata.novel_title,
-                                        novel_base_dir=novel_base_dir)
 
         if toc_html:
             self.file_manager.add_toc(toc_html)
@@ -155,9 +149,10 @@ class Novel:
             sys.exit(1)
 
         self.host = host if host else utils.obtain_host(self.toc_main_url)
-        self.decoder = Decoder(self.host)
 
-        self.save_novel()
+        self.config = None
+        self.file_manager = None
+        self.decoder = None
 
     def __str__(self):
         """
@@ -165,7 +160,7 @@ class Novel:
         """
         toc_info = self.toc_main_url if self.toc_main_url else "TOC added manually"
         attributes = [
-            f"Title: {self.metadata.novel_title}",
+            f"Title: {self.title}",
             f"Author: {self.metadata.author}",
             f"Language: {self.metadata.language}",
             f"Description: {self.metadata.description}",
@@ -177,30 +172,57 @@ class Novel:
         return (f"Novel Info: \n"
                 f"{attributes_str}")
 
+    @staticmethod
+    def load(title: str, cfg: ScraperConfig, novel_base_dir: str | None = None):
+        fm = FileManager(title, cfg.base_novels_dir, novel_base_dir, read_only=True)
+        raw = fm.load_novel_json()
+        if raw is None:
+            logger.debug(f'Novel "{title}" was not found.')
+            raise ValueError(f'Novel "{title}" was not found.')
+        novel = Novel.from_json(raw)
+        novel.config = cfg
+        novel.set_config(cfg=cfg, novel_base_dir=novel_base_dir)
+        return novel
+
     # NOVEL PARAMETERS MANAGEMENT
 
-    def set_scraper_behavior(self, **kwargs) -> None:
+    def set_config(self,
+                   cfg: ScraperConfig = None,
+                   config_file: str = None,
+                   base_novels_dir: str = None,
+                   novel_base_dir: str = None,
+                   decode_guide_file: str = None):
+        if cfg is not None:
+            self.config = cfg
+        else:
+            self.config = ScraperConfig(config_file=config_file,
+                                        base_novels_dir=base_novels_dir,
+                                        decode_guide_file=decode_guide_file)
+
+        self.file_manager = FileManager(title=self.title,
+                                        base_novels_dir=self.config.base_novels_dir,
+                                        novel_base_dir=novel_base_dir)
+
+        self.decoder = Decoder(self.host, self.config.decode_guide_file)
+
+    def set_scraper_behavior(self, save: bool = False, **kwargs) -> None:
         self.scraper_behavior.update_behavior(**kwargs)
-        self.save_novel()
 
     def set_metadata(self, **kwargs) -> None:
         self.metadata.update_behavior(**kwargs)
-        self.save_novel()
 
     def add_tag(self, tag: str) -> bool:
         if tag not in self.metadata.tags:
             self.metadata.tags.append(tag)
-            self.save_novel()
             return True
-        logger.warning(f'Tag "{tag}" already exists on novel {self.metadata.novel_title}')
+        logger.warning(f'Tag "{tag}" already exists on novel {self.title}')
         return False
 
     def remove_tag(self, tag: str) -> bool:
         if tag in self.metadata.tags:
             self.metadata.tags.remove(tag)
-            self.save_novel()
             return True
-        logger.warning(f'Tag "{tag}" doesn\'t exist on novel {self.metadata.novel_title}')
+        logger.warning(f'Tag "{tag}" doesn\'t exist on novel {self.title}')
         return False
 
     def set_cover_image(self, cover_image_path: str) -> bool:
@@ -208,10 +230,9 @@ class Novel:
 
     def set_host(self, host: str) -> None:
         self.host = host
-        self.decoder = Decoder(self.host)
-        self.save_novel()
+        self.decoder.set_host(host)
 
-    def save_novel(self) -> None:
+    def save_novel(self, save: bool = True) -> None:
         self.file_manager.save_novel_json(self.to_dict())
 
     # TABLE OF CONTENTS MANAGEMENT
@@ -224,7 +245,6 @@ class Novel:
             self.decoder = Decoder(self.host)
         elif update_host:
             self.decoder = Decoder(utils.obtain_host(self.toc_main_url))
-        self.save_novel()
 
     def add_toc_html(self, html: str, host: str = None) -> None:
         if self.toc_main_url:
@@ -236,13 +256,11 @@ class Novel:
             self.decoder = Decoder(self.host)
         self.file_manager.add_toc(html)
         # Delete toc_main_url since they are exclusive
-        self.save_novel()
 
     def delete_toc(self):
         self.file_manager.delete_toc()
         self.chapters = []
         self.chapters_url_list = []
-        self.save_novel()
 
     def sync_toc(self, reload_files: bool = False) -> bool:
         # Hard reload will request again the toc files from the toc_main_url
@@ -286,8 +304,8 @@ class Novel:
             for chapters_url in (self.chapters_url_list[::-1] if invert else self.chapters_url_list)
             for chapter in chapters_url
         ]
-
-        if self.scraper_behavior.auto_add_host:
+        add_host_to_chapter = self.scraper_behavior.auto_add_host or self.decoder.add_host_to_chapter()
+        if add_host_to_chapter:
             self.chapters_url_list = [
                 f'https://{self.host}{chapter_url}' for chapter_url in self.chapters_url_list]
         self.chapters_url_list = utils.delete_duplicates(
@@ -337,6 +355,7 @@ class Novel:
             chapter = self.chapters[chapter_idx]
         if update_html:
             logger.debug('HTML will be updated...')
+
         chapter = self._get_chapter(chapter,
                                     reload=update_html)
 
@@ -437,7 +456,7 @@ class Novel:
         return True
 
 
-# UTILS
+    ## UTILS
 
 
     def clean_files(self, clean_chapters: bool = True, clean_toc: bool = True, hard_clean: bool = False) -> None:
@@ -453,6 +472,9 @@ class Novel:
     def show_novel_dir(self) -> str:
         return self.file_manager.novel_base_dir
 
+
+    ## PRIVATE HELPERS
+
     def _clean_chapter(self, chapter_html_filename: str, hard_clean: bool = False) -> None:
         hard_clean = hard_clean or self.scraper_behavior.hard_clean
         chapter_html = self.file_manager.load_chapter_html(
@@ -600,13 +622,14 @@ class Novel:
         chapter_title = self.decoder.get_chapter_title(chapter.chapter_html)
         if not chapter_title:
             logger.debug('No chapter title found, generating one...')
-            chapter_title = f'{self.metadata.novel_title} Chapter {idx_for_chapter_name}'
+            chapter_title = f'{self.title} Chapter {idx_for_chapter_name}'
         chapter.chapter_title = str(chapter_title)
         logger.debug(f'Chapter title: "{chapter_title}"')
 
         logger.debug('Obtaining chapter content...')
+        save_title_to_content = self.scraper_behavior.save_title_to_content or self.decoder.save_title_to_content()
         chapter.chapter_content = self.decoder.get_chapter_content(chapter.chapter_html,
-                                                           self.scraper_behavior.save_title_to_content,
+                                                           save_title_to_content,
                                                            chapter.chapter_title)
         logger.debug('Chapter successfully decoded')
 
@@ -615,7 +638,7 @@ class Novel:
     def _create_epub_book(self, book_title: str = None, calibre_collection: dict = None) -> epub.EpubBook:
         book = epub.EpubBook()
         if not book_title:
-            book_title = self.metadata.novel_title
+            book_title = self.title
         book.set_title(book_title)
         book.set_language(self.metadata.language)
         book.add_metadata('DC', 'description', self.metadata.description)
@@ -700,11 +723,11 @@ class Novel:
         idx_start = start_chapter - 1
         idx_end = end_chapter
         # We create the epub book
-        book_title = f'{self.metadata.novel_title} Chapters {start_chapter} - {end_chapter}'
+        book_title = f'{self.title} Chapters {start_chapter} - {end_chapter}'
         calibre_collection = None
         # If collection_idx is set, we create a calibre collection
         if collection_idx:
-            calibre_collection = {'title': self.metadata.novel_title,
+            calibre_collection = {'title': self.title,
                                   'idx': str(collection_idx)}
         book = self._create_epub_book(book_title, calibre_collection)
 

web_novel_scraper/request_manager.py

@@ -45,7 +45,7 @@ def _get_request(url: str,
         if attempt < retries - 1:
             logger.debug(f'Waiting {time_between_retries} seconds before retrying')
             time.sleep(time_between_retries)  # Wait before retrying
-    logger.warning(f'Failed to get a successful response for "{url}" after {retries} attempts using common HTTP Request')
+    logger.debug(f'Failed to get a successful response for "{url}" after {retries} attempts using common HTTP Request')
     return None
 
 
@@ -96,7 +96,7 @@ def _get_request_flaresolver(url: str,
             logger.debug(f'Waiting {time_between_retries} seconds before retrying')
             time.sleep(time_between_retries)  # Wait before retrying
 
-    logger.warning(f'Failed to get a successful response for "{url}" using FlareSolver after {retries} attempts')
+    logger.debug(f'Failed to get a successful response for "{url}" using FlareSolver after {retries} attempts')
     return None
 
 

web_novel_scraper/utils.py

@@ -1,10 +1,140 @@
-from .file_manager import FileManager
-from . import request_manager
+import json
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
 import hashlib
 from urllib.parse import urlparse
 import re
 import unicodedata
 
+class FileManagerError(Exception):
+    """Exception raised for any exception for file operations"""
+
+class FileOps:
+    """Static helper for disc operations."""
+
+    ## HELPERS
+
+    @staticmethod
+    def _atomic_tmp(path: Path) -> Path:
+        """Temporary file path in the same directory as *path*."""
+        return path.with_suffix(path.suffix + ".tmp")
+
+    ## DIRECTORY MANAGEMENT
+    @staticmethod
+    def ensure_dir(path: Path) -> Path:
+        """Create *path* (and parents) if missing."""
+        try:
+            path.mkdir(parents=True, exist_ok=True)
+            return path
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+    ## READ OPERATIONS
+
+    @staticmethod
+    def read_text(path: Path) -> Optional[str]:
+        """Return UTF-8 contents or None if *path* does not exist."""
+        if not path.exists():
+            return None
+        try:
+            return path.read_text(encoding="utf-8")
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+    @staticmethod
+    def read_json(path: Path | str) -> Optional[dict]:
+        """Return JSON object or None if *path* does not exist."""
+        path = Path(path)
+        raw = FileOps.read_text(path)
+        if raw is None:
+            return None
+        try:
+            return json.loads(raw)
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+    @staticmethod
+    def read_binary(path: Path) -> Optional[bytes]:
+        """Return binary contents or None if *path* does not exist."""
+        if not path.exists():
+            return None
+        try:
+            return path.read_bytes()
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+    ## WRITE OPERATION
+
+    @staticmethod
+    def save_text(path: Path, text: str) -> None:
+        """Atomically write UTF-8 text to *path*."""
+        tmp = FileOps._atomic_tmp(path)
+        try:
+            tmp.write_text(text, encoding="utf-8")
+            tmp.replace(path)
+        except Exception as e:
+            FileOps.delete(tmp)
+            raise FileManagerError(str(e)) from e
+
+    @staticmethod
+    def save_json(path: Path, obj: dict) -> None:
+        """Atomically write pretty-printed JSON to *path*."""
+        tmp = FileOps._atomic_tmp(path)
+        try:
+            tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding="utf-8")
+            tmp.replace(path)
+        except Exception as e:
+            FileOps.delete(tmp)
+            raise FileManagerError(str(e)) from e
+
+    @staticmethod
+    def save_binary(path: Path, data: bytes) -> None:
+        """Atomically write binary data to *path* (e.g., cover images)."""
+        tmp = FileOps._atomic_tmp(path)
+        try:
+            tmp.write_bytes(data)
+            tmp.replace(path)
+        except Exception as e:
+            FileOps.delete(tmp)
+            raise FileManagerError(str(e)) from e
+
+    ## DELETE/COPY OPERATIONS
+
+    @staticmethod
+    def delete(path: Path) -> None:
+        """Delete *path* if it exists."""
+        try:
+            if path.exists():
+                path.unlink()
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+    @staticmethod
+    def copy(src: Path, dst: Path) -> None:
+        """Copy *src* to *dst*."""
+        try:
+            shutil.copy(src, dst)
+        except Exception as e:
+            raise FileManagerError(str(e)) from e
+
+def _normalize_dirname(name: str) -> str:
+    """
+    Keep whitespace as-is while replacing any other unsupported characters
+    with an underscore.
+    Allowed: letters, digits, underscore, hyphen, and spaces.
+    """
+    # Collapse multiple spaces into a single space (optional; comment out if not desired)
+    name = re.sub(r'\s+', ' ', name.strip())
+
+    # Replace any char that is *not* letter, digit, underscore, hyphen, or space.
+    return re.sub(r'[^\w\-\s]', '_', name)
+
+def now_iso() -> str:
+    """Current timestamp in ISO-8601 (seconds precision)."""
+    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
 
 def generate_file_name_from_url(url: str) -> str:
     # Parsea URL

web_novel_scraper/version.py

@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "2.0.0"

{web_novel_scraper-1.1.1.dist-info → web_novel_scraper-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-novel-scraper
-Version: 1.1.1
+Version: 2.0.0
 Summary: Python tool that allows you to scrape web novels from various sources and save them to more readable formats like EPUB.
 Project-URL: Homepage, https://github.com/ImagineBrkr/web-novel-scraper
 Project-URL: Documentation, https://web-novel-scraper.readthedocs.io

web_novel_scraper-2.0.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+web_novel_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+web_novel_scraper/__main__.py,sha256=wWfOsHR_nswlWTj_3aZ11vJXzqua2Va_R7h6wyS4XWc,18266
+web_novel_scraper/config_manager.py,sha256=duwKc6jyLj8NmST5F98qGgpW_o6D6GAenKWsYQ80gcU,3121
+web_novel_scraper/decode.py,sha256=VlAW3bdWCSC2BloTIKJ74KqlBhzUca9vpvYZVd2dziI,10796
+web_novel_scraper/file_manager.py,sha256=zmkeJWh8PoVcjnA38qZRSYH06kh0i0txFE-1bfa5vKg,10598
+web_novel_scraper/logger_manager.py,sha256=A-a4bhYI4YCEuSJd9E3WH_kanJ7YCASMwheBzObZK4Q,1972
+web_novel_scraper/novel_scraper.py,sha256=dhiRC1h5Q0b4UrU-_0n2RfH2qRP-c7hIPJukAHNwWsQ,30532
+web_novel_scraper/request_manager.py,sha256=BVWMtUO3HRs44phU2ODkPUjy7tJiIBX_R0rxGGLaJzw,6617
+web_novel_scraper/utils.py,sha256=1V8UwYhpp_27zqPnBDbl7fohu2Z7Sy_4Fq5J2_JAEvU,6405
+web_novel_scraper/version.py,sha256=_7OlQdbVkK4jad0CLdpI0grT-zEAb-qgFmH5mFzDXiA,22
+web_novel_scraper/custom_processor/__init__.py,sha256=iy4tjivMjshSzc52--aa-jK53qu9VwdK-6p4vuQc6oc,103
+web_novel_scraper/custom_processor/custom_processor.py,sha256=h1MPl6JU_C2Mc7SqK70LsNQHpDzSL6QyraMIQ87HcMM,870
+web_novel_scraper/custom_processor/sites/genesis.py,sha256=xV0eybI0ieHR5gn4yWXI74l99Eayhqs16PIYs-BrPjE,1843
+web_novel_scraper/custom_processor/sites/royalroad.py,sha256=_2PsFC_w3RJCUkAPoRn-7R2jlzl3XsG4WYtRaQkp6lg,787
+web_novel_scraper/decode_guide/decode_guide.json,sha256=kbPjEYZCN_jEpV5CL8v22yZ_4gPkO56XwblkSGDMEeM,9683
+web_novel_scraper-2.0.0.dist-info/METADATA,sha256=BFByjX9R_QUaRpcbfEjPEt_UmcZq5M_AXSCDo7nbzVA,8423
+web_novel_scraper-2.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+web_novel_scraper-2.0.0.dist-info/entry_points.txt,sha256=bqRvStfvSprSJc2EJXgKIbggWOXSePHFfVIZWy_plDQ,69
+web_novel_scraper-2.0.0.dist-info/RECORD,,

web_novel_scraper-1.1.1.dist-info/RECORD

@@ -1,18 +0,0 @@
-web_novel_scraper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-web_novel_scraper/__main__.py,sha256=OQQVX5CttmAkUwdrnjBSjKPaoh_boUI2ysHi3rLGOSs,17769
-web_novel_scraper/decode.py,sha256=U-78PhJ4SU2hiUmfAWeWGEBJ3YSoCW3Lupw9cUqQuI0,11013
-web_novel_scraper/file_manager.py,sha256=qAqgqtaRb7QyVtyEOW2cMhPYWdKM6nJ69weUCYKwVtM,11862
-web_novel_scraper/logger_manager.py,sha256=A-a4bhYI4YCEuSJd9E3WH_kanJ7YCASMwheBzObZK4Q,1972
-web_novel_scraper/novel_scraper.py,sha256=DsYnY15s8cZZ2w8pRvmD3_NJw54xarhcnEQdvnTD8XI,29421
-web_novel_scraper/request_manager.py,sha256=WU8LG6D_fqmDapX6wpVwpQQSItcNU8Qb9dMAlLCYI8U,6621
-web_novel_scraper/utils.py,sha256=dPp7D2ji9mC2nFydqxsJ_9vkAntA_3VTt8ZmG-F1f78,2270
-web_novel_scraper/version.py,sha256=q8_5C0f-8mHWNb6mMw02zlYPnEGXBqvOmP3z0CEwZKM,22
-web_novel_scraper/custom_processor/__init__.py,sha256=iy4tjivMjshSzc52--aa-jK53qu9VwdK-6p4vuQc6oc,103
-web_novel_scraper/custom_processor/custom_processor.py,sha256=h1MPl6JU_C2Mc7SqK70LsNQHpDzSL6QyraMIQ87HcMM,870
-web_novel_scraper/custom_processor/sites/genesis.py,sha256=xV0eybI0ieHR5gn4yWXI74l99Eayhqs16PIYs-BrPjE,1843
-web_novel_scraper/custom_processor/sites/royalroad.py,sha256=_2PsFC_w3RJCUkAPoRn-7R2jlzl3XsG4WYtRaQkp6lg,787
-web_novel_scraper/decode_guide/decode_guide.json,sha256=gNVencLtK0HmZPlubTm1wA7eatWADCxJ_LCOYWHWuA0,8556
-web_novel_scraper-1.1.1.dist-info/METADATA,sha256=ow5piBhzzo4mZ0secvHrqc4KCCt4VInpDa09Qo9l4AE,8423
-web_novel_scraper-1.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-web_novel_scraper-1.1.1.dist-info/entry_points.txt,sha256=bqRvStfvSprSJc2EJXgKIbggWOXSePHFfVIZWy_plDQ,69
-web_novel_scraper-1.1.1.dist-info/RECORD,,