novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/utils/network.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.network
+------------------------------
+
+Utilities for handling HTTP requests and downloading remote resources.
+"""
+
+import logging
+import random
+import time
+from pathlib import Path
+from typing import Dict, Literal, Optional, Union
+from urllib.parse import unquote, urlparse
+
+import requests
+
+from .constants import DEFAULT_HEADERS, DEFAULT_IMAGE_SUFFIX
+from .file_utils.io import _get_non_conflicting_path, _write_file, read_binary_file
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_CHUNK_SIZE = 8192  # 8KB per chunk for streaming downloads
+
+
+def http_get_with_retry(
+    url: str,
+    *,
+    retries: int = 3,
+    timeout: int = 10,
+    backoff: float = 0.5,
+    headers: Optional[Dict[str, str]] = None,
+    stream: bool = False,
+) -> Optional[requests.Response]:
+    """
+    Perform a GET request with retry support.
+
+    :param url: URL to request.
+    :param retries: Number of retry attempts.
+    :param timeout: Timeout in seconds per request.
+    :param backoff: Base backoff delay between retries.
+    :param headers: Optional HTTP headers.
+    :param stream: Whether to stream the response.
+    :return: Response object if successful, else None.
+    """
+    for attempt in range(1, retries + 1):
+        try:
+            response = requests.get(
+                url, timeout=timeout, headers=headers, stream=stream
+            )
+            response.raise_for_status()
+            return response
+        except requests.RequestException as e:
+            logger.warning("[http] Attempt %s/%s failed: %s", attempt, retries, e)
+            if attempt < retries:
+                sleep_time = backoff * (2 ** (attempt - 1)) + random.uniform(0, 0.1)
+                time.sleep(sleep_time)
+        except Exception as e:
+            logger.error("[http] Unexpected error: %s", e)
+            break
+
+    logger.error("[http] Failed after %s attempts: %s", retries, url)
+    return None
+
+
+def image_url_to_filename(url: str) -> str:
+    """
+    Parse and sanitize a image filename from a URL.
+    If no filename or suffix exists, fallback to default name and extension.
+
+    :param url: URL string
+    :return: Safe filename string
+    """
+    parsed_url = urlparse(url)
+    path = unquote(parsed_url.path)
+    filename = Path(path).name
+
+    if not filename:
+        filename = "image"
+
+    if not Path(filename).suffix:
+        filename += DEFAULT_IMAGE_SUFFIX
+
+    return filename
+
+
+def download_image_as_bytes(
+    url: str,
+    target_folder: Optional[Union[str, Path]] = None,
+    *,
+    timeout: int = 10,
+    retries: int = 3,
+    backoff: float = 0.5,
+    on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
+) -> Optional[bytes]:
+    """
+    Download an image from a given URL and return its content as bytes.
+
+    If on_exist='skip' and the file already exists, it will be read from disk
+    instead of being downloaded again.
+
+    :param url: Image URL. Can start with 'http', '//', or without protocol.
+    :param target_folder: Optional folder to save the image (str or Path).
+    :param timeout: Request timeout in seconds.
+    :param retries: Number of retry attempts.
+    :param backoff: Base delay between retries (exponential backoff).
+    :param on_exist: What to do if file exists: 'overwrite', 'skip', or 'rename'.
+    :return: Image content as bytes, or None if failed.
+    """
+    # Normalize URL
+    if url.startswith("//"):
+        url = "https:" + url
+    elif not url.startswith("http"):
+        url = "https://" + url
+
+    save_path = None
+    if target_folder:
+        target_folder = Path(target_folder)
+        filename = image_url_to_filename(url)
+        save_path = target_folder / filename
+
+        if on_exist == "skip" and save_path.exists():
+            logger.info(
+                "[image] '%s' exists, skipping download and reading from disk.",
+                save_path,
+            )
+            return read_binary_file(save_path)
+
+    # Proceed with download
+    response = http_get_with_retry(
+        url,
+        retries=retries,
+        timeout=timeout,
+        backoff=backoff,
+        headers=DEFAULT_HEADERS,
+        stream=False,
+    )
+
+    if response and response.ok:
+        content = response.content
+
+        if save_path:
+            _write_file(
+                content=content,
+                filepath=save_path,
+                mode="wb",
+                on_exist=on_exist,
+            )
+
+        return content
+
+    return None
+
+
+def download_font_file(
+    url: str,
+    target_folder: Union[str, Path],
+    *,
+    timeout: int = 10,
+    retries: int = 3,
+    backoff: float = 0.5,
+    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
+) -> Optional[Path]:
+    """
+    Download a font file from a URL and save it locally with retry and overwrite control
+
+    :param url: Fully-qualified font file URL.
+    :param target_folder: Local folder to save the font file.
+    :param timeout: Timeout for each request (in seconds).
+    :param retries: Number of retry attempts.
+    :param backoff: Base backoff time between retries (in seconds).
+    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
+    :return: Path to the saved font file, or None if failed.
+    """
+    # Validate and parse URL
+    parsed = urlparse(url)
+    if not parsed.scheme or not parsed.netloc:
+        logger.warning("[font] Invalid URL: %s", url)
+        return None
+
+    # Determine filename
+    filename = Path(unquote(parsed.path)).name
+    if not filename:
+        logger.warning("[font] Could not extract filename from URL: %s", url)
+        return None
+
+    # Resolve save path
+    target_folder = Path(target_folder)
+    target_folder.mkdir(parents=True, exist_ok=True)
+    font_path = target_folder / filename
+
+    # If skip and file exists -> return immediately
+    if on_exist == "skip" and font_path.exists():
+        logger.info("[font] File exists, skipping download: %s", font_path)
+        return font_path
+
+    # Retry download with exponential backoff
+    response = http_get_with_retry(
+        url,
+        retries=retries,
+        timeout=timeout,
+        backoff=backoff,
+        headers=DEFAULT_HEADERS,
+        stream=True,
+    )
+
+    if response:
+        try:
+            if on_exist == "rename":
+                font_path = _get_non_conflicting_path(font_path)
+
+            with open(font_path, "wb") as f:
+                for chunk in response.iter_content(chunk_size=_DEFAULT_CHUNK_SIZE):
+                    if chunk:
+                        f.write(chunk)
+
+            logger.info("[font] Font saved to: %s", font_path)
+            return font_path
+
+        except Exception as e:
+            logger.error("[font] Error writing font to disk: %s", e)
+
+    return None
+
+
+def download_js_file(
+    url: str,
+    target_folder: Union[str, Path],
+    *,
+    timeout: int = 10,
+    retries: int = 3,
+    backoff: float = 0.5,
+    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
+) -> Optional[Path]:
+    """
+    Download a JavaScript (.js) file from a URL and save it locally.
+
+    :param url: Fully-qualified JS file URL.
+    :param target_folder: Local folder to save the JS file.
+    :param timeout: Timeout for each request (in seconds).
+    :param retries: Number of retry attempts.
+    :param backoff: Base backoff time between retries (in seconds).
+    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
+    :return: Path to the saved JS file, or None if failed.
+    """
+    parsed = urlparse(url)
+    if not parsed.scheme or not parsed.netloc:
+        logger.warning("[js] Invalid URL: %s", url)
+        return None
+
+    # Determine filename
+    filename = Path(unquote(parsed.path)).name
+    if not filename.endswith(".js"):
+        filename += ".js"
+
+    target_folder = Path(target_folder)
+    target_folder.mkdir(parents=True, exist_ok=True)
+    save_path = target_folder / filename
+
+    if on_exist == "skip" and save_path.exists():
+        logger.info("[js] File exists, skipping download: %s", save_path)
+        return save_path
+
+    response = http_get_with_retry(
+        url,
+        retries=retries,
+        timeout=timeout,
+        backoff=backoff,
+        headers=DEFAULT_HEADERS,
+        stream=False,
+    )
+
+    if response and response.ok:
+        content = response.content
+
+        if on_exist == "rename":
+            save_path = _get_non_conflicting_path(save_path)
+
+        try:
+            _write_file(content=content, filepath=save_path, mode="wb")
+            logger.info("[js] JS file saved to: %s", save_path)
+            return save_path
+        except Exception as e:
+            logger.error("[js] Error writing JS to disk: %s", e)
+
+    return None

novel_downloader/utils/state.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.state
+----------------------------
+State management for user preferences and runtime flags.
+
+Supported sections:
+- general: global preferences (e.g. language)
+- sites: per-site flags & data (e.g. manual_login, cookies)
+"""
+import json
+from pathlib import Path
+from typing import Any, Dict, Union
+
+from .constants import STATE_FILE
+
+
+class StateManager:
+    """
+    Manages persistent state for user preferences and runtime flags.
+    Stores data in JSON at STATE_FILE.
+    """
+
+    def __init__(self, path: Path = STATE_FILE) -> None:
+        self._path = path
+        self._data = self._load()
+
+    def _load(self) -> Dict[str, Any]:
+        """
+        Load the configuration file into a Python dictionary.
+
+        :return: A dict representing the full config state.
+        """
+        if not self._path.exists():
+            return {}
+        try:
+            text = self._path.read_text(encoding="utf-8")
+            return json.loads(text) or {}
+        except Exception:
+            return {}
+
+    def _save(self) -> None:
+        """
+        Save a configuration dictionary to the config file.
+
+        :param data: A dict representing the full config state to be written.
+        """
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        content = json.dumps(self._data, ensure_ascii=False, indent=2)
+        self._path.write_text(content, encoding="utf-8")
+
+    def _parse_cookie_string(self, cookie_str: str) -> Dict[str, str]:
+        """
+        Parse a Cookie header string into a dict.
+
+        :param cookie_str: e.g. 'k1=v1; k2=v2; k3'
+        :return: mapping cookie names to values (missing '=' yields empty string)
+        :rtype: Dict[str, str]
+        """
+        cookies: Dict[str, str] = {}
+        for item in cookie_str.split(";"):
+            item = item.strip()
+            if not item:
+                continue
+            if "=" in item:
+                k, v = item.split("=", 1)
+                cookies[k.strip()] = v.strip()
+            else:
+                cookies[item] = ""
+        return cookies
+
+    def get_language(self) -> str:
+        """
+        Load the user's language preference, defaulting to 'zh'.
+
+        :return: Language code string
+        """
+        lang = self._data.get("general", {}).get("lang", "zh")
+        return str(lang)
+
+    def set_language(self, lang: str) -> None:
+        """
+        Save the user's language preference.
+
+        :param lang: Language code (e.g. 'zh', 'en')
+        """
+        self._data.setdefault("general", {})["lang"] = lang
+        self._save()
+
+    def get_manual_login_flag(self, site: str) -> bool:
+        """
+        Retrieve the manual login requirement flag for a specific site.
+
+        :param site: Site identifier (e.g. 'qidian', 'bqg')
+        :return: True if manual login is required (defaults to True)
+        """
+        val = self._data.get("sites", {}).get(site, {}).get("manual_login", True)
+        return bool(val)
+
+    def set_manual_login_flag(self, site: str, flag: bool) -> None:
+        """
+        Set the 'manual_login' flag for a specific site.
+
+        :param flag: True if the site requires manual login.
+        :param site: Site identifier (e.g. 'qidian', 'bqg')
+        """
+        sites = self._data.setdefault("sites", {})
+        site_data = sites.setdefault(site, {})
+        site_data["manual_login"] = flag
+        self._save()
+
+    def get_cookies(self, site: str) -> Dict[str, str]:
+        """
+        Retrieve the persisted cookies for a specific site.
+
+        :param site: Site identifier (e.g. 'qidian', 'bqg')
+        :return: A dict mapping cookie names to values. Returns empty dict if not set.
+        """
+        cookies = self._data.get("sites", {}).get(site, {}).get("cookies", {})
+        return {str(k): str(v) for k, v in cookies.items()}
+
+    def set_cookies(self, site: str, cookies: Union[str, Dict[str, str]]) -> None:
+        """
+        Persist (overwrite) the cookies for a specific site.
+
+        :param site: Site identifier (e.g. 'qidian', 'bqg')
+        :param cookies: Either a dict mapping cookie names to values,
+                        or a string (JSON or 'k=v; k2=v2') to be parsed.
+        :raises TypeError: if cookies is neither str nor dict
+        """
+        # 1) normalize to dict
+        if isinstance(cookies, dict):
+            cookies_dict = cookies
+        elif isinstance(cookies, str):
+            # try JSON first
+            try:
+                parsed = json.loads(cookies)
+                if isinstance(parsed, dict):
+                    cookies_dict = parsed  # OK!
+                else:
+                    raise ValueError
+            except Exception:
+                # fallback to "k=v; k2=v2" format
+                cookies_dict = self._parse_cookie_string(cookies)
+        else:
+            raise TypeError("`cookies` must be a dict or a str")
+
+        # 2) persist
+        sites = self._data.setdefault("sites", {})
+        site_data = sites.setdefault(site, {})
+        site_data["cookies"] = {str(k): str(v) for k, v in cookies_dict.items()}
+        self._save()
+
+
+state_mgr = StateManager()

novel_downloader/utils/text_utils/__init__.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.text_utils
+---------------------------------
+
+Utility modules for text formatting, font mapping, cleaning, and diff display.
+
+Submodules:
+- font_mapping: Replace obfuscated characters using font maps
+- chapter_formatting: Build structured chapter strings from raw content
+- text_cleaning: Remove promo text and check for spam lines
+- diff_display: Generate inline diffs with aligned character markers
+"""
+
+from .chapter_formatting import format_chapter
+from .diff_display import diff_inline_display
+from .font_mapping import apply_font_mapping
+from .text_cleaning import clean_chapter_title, is_promotional_line
+
+__all__ = [
+    "apply_font_mapping",
+    "format_chapter",
+    "clean_chapter_title",
+    "is_promotional_line",
+    "diff_inline_display",
+]

novel_downloader/utils/text_utils/chapter_formatting.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.text_utils.chapter_formatting
+----------------------------------------------------
+
+Format chapter content with title, paragraph blocks, and optional author notes.
+"""
+
+from typing import List, Optional
+
+
+def format_chapter(
+    title: str, paragraphs: str, author_say: Optional[str] = None
+) -> str:
+    """
+    Build a formatted chapter string with title, paragraphs, and optional author note.
+
+    :param title:       The chapter title.
+    :param paragraphs:  Raw multi‐line string; lines are treated as paragraphs.
+    :param author_say:  Optional author comment to append at the end.
+    :return:            A single string where title, paragraphs, and author note
+                        are separated by blank lines.
+    """
+    parts: List[str] = [title.strip()]
+
+    # add each nonempty paragraph line
+    for ln in paragraphs.splitlines():
+        line = ln.strip()
+        if line:
+            parts.append(line)
+
+    # add author_say lines if present
+    if author_say:
+        author_lines = [ln.strip() for ln in author_say.splitlines() if ln.strip()]
+        if author_lines:
+            parts.append("---")
+            parts.append("作者说:")
+            parts.extend(author_lines)
+
+    return "\n\n".join(parts)
+
+
+__all__ = [
+    "format_chapter",
+]

novel_downloader/utils/text_utils/diff_display.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.text_utils.diff_display
+----------------------------------------------
+
+Generate inline character-level diff between two strings with visual markers.
+"""
+
+import difflib
+import unicodedata
+
+
+def _char_width_space(
+    c: str, normal_char: str = " ", asian_char: str = "\u3000"
+) -> str:
+    """
+    Return a placeholder space character matching the width of `c`.
+
+    Fullwidth (F) or Wide (W) characters map to `asian_char`, else `normal_char`.
+
+    :param c:           A single character.
+    :param normal_char: Replacement for narrow chars (default U+0020).
+    :param asian_char:  Replacement for wide chars (default U+3000).
+    :return:            The appropriate space character.
+    """
+    return asian_char if unicodedata.east_asian_width(c) in ("F", "W") else normal_char
+
+
+def diff_inline_display(old_str: str, new_str: str) -> str:
+    """
+    Show inline diff between two strings,
+    marking deletions with '^' and insertions with '^'.
+
+    :param old_str: Original string (prefixed '-' will be trimmed).
+    :param new_str: Modified string (prefixed '+' will be trimmed).
+    :return:        A multiline diff display with aligned markers.
+    """
+    space_1 = " "
+    space_2 = "\u3000"
+    mark_1 = "^"
+    mark_2 = "\ufe3f"  # '人' / '\ufe3f' / '宀' / '立' / '八'
+
+    # Clean leading +/- if present
+    s1 = old_str.lstrip("-").strip()
+    s2 = new_str.lstrip("+").strip()
+
+    sm = difflib.SequenceMatcher(None, s1, s2)
+    marker_s1 = ""
+    marker_s2 = ""
+
+    for tag, i1, i2, j1, j2 in sm.get_opcodes():
+        if tag == "equal":
+            s1_seg = s1[i1:i2]
+            s2_seg = s2[j1:j2]
+            marker_s1 += "".join(_char_width_space(c, space_1, space_2) for c in s1_seg)
+            marker_s2 += "".join(_char_width_space(c, space_1, space_2) for c in s2_seg)
+        elif tag == "delete":
+            seg = s1[i1:i2]
+            marker_s1 += "".join(_char_width_space(c, mark_1, mark_2) for c in seg)
+        elif tag == "insert":
+            seg = s2[j1:j2]
+            marker_s2 += "".join(_char_width_space(c, mark_1, mark_2) for c in seg)
+        elif tag == "replace":
+            s1_seg = s1[i1:i2]
+            s2_seg = s2[j1:j2]
+            marker_s1 += "".join(_char_width_space(c, mark_1, mark_2) for c in s1_seg)
+            marker_s2 += "".join(_char_width_space(c, mark_1, mark_2) for c in s2_seg)
+    output_str = f"-{s1}\n {marker_s1}\n+{s2}\n {marker_s2}"
+    return output_str
+
+
+__all__ = [
+    "diff_inline_display",
+]

novel_downloader/utils/text_utils/font_mapping.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.text_utils.font_mapping
+----------------------------------------------
+
+Utility for decoding obfuscated text by applying character-level font mapping.
+
+This is commonly used to reverse font-based obfuscation in scraped content,
+where characters are visually disguised via custom font glyphs but can be
+recovered using a known mapping.
+"""
+
+from typing import Dict
+
+
+def apply_font_mapping(text: str, font_map: Dict[str, str]) -> str:
+    """
+    Replace each character in `text` using `font_map`,
+    leaving unmapped characters unchanged.
+
+    :param text:    The input string, possibly containing obfuscated font chars.
+    :param font_map: A dict mapping obfuscated chars to real chars.
+    :return:        The de‐obfuscated text.
+    """
+    return "".join(font_map.get(ch, ch) for ch in text)
+
+
+__all__ = [
+    "apply_font_mapping",
+]

novel_downloader/utils/text_utils/text_cleaning.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.utils.text_utils.text_cleaning
+-----------------------------------------------
+
+Tools for detecting and removing promotional or ad-like content from text.
+"""
+
+import re
+
+from novel_downloader.utils.file_utils.io import load_blacklisted_words
+
+# --- Constants & Precompiled Patterns ---
+
+_BLACKLISTED_WORDS = load_blacklisted_words()
+
+_BRACKET_PATTERN = re.compile(r"[\(（](.*?)[\)）]")
+_K_PROMO_PATTERN = re.compile(r"\b\d{1,4}k\b", re.IGNORECASE)
+
+
+def clean_chapter_title(title: str) -> str:
+    """
+    Remove bracketed promotional content from a chapter title.
+
+    If any blacklisted word appears inside parentheses (Chinese or English),
+    the entire bracketed section is stripped.
+
+    :param title: Original title, possibly containing ad text in brackets.
+    :return:      Title with offending bracketed sections removed.
+    """
+    cleaned = title
+    for content in _BRACKET_PATTERN.findall(title):
+        if any(bw in content for bw in _BLACKLISTED_WORDS):
+            cleaned = re.sub(rf"[\(（]{re.escape(content)}[\)）]", "", cleaned)
+    return cleaned.strip()
+
+
+def is_promotional_line(line: str) -> bool:
+    """
+    Check if a line of text likely contains promotional or ad‐like content.
+
+    :param line: A single line of text.
+    :return:     True if it contains promo keywords or a '###k' vote count pattern.
+    """
+    low = line.lower()
+    if any(kw in low for kw in _BLACKLISTED_WORDS):
+        return True
+    if _K_PROMO_PATTERN.search(low):
+        return True
+    return False
+
+
+__all__ = [
+    "clean_chapter_title",
+    "is_promotional_line",
+]