novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/utils/network.py

@@ -6,295 +6,154 @@ novel_downloader.utils.network
 Utilities for handling HTTP requests and downloading remote resources.
 """
 
+__all__ = ["download"]
+
 import logging
-import random
-import time
 from pathlib import Path
 from typing import Literal
 from urllib.parse import unquote, urlparse
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
-from .constants import DEFAULT_HEADERS, DEFAULT_IMAGE_SUFFIX
-from .file_utils.io import _get_non_conflicting_path, _write_file
+from .constants import DEFAULT_HEADERS
+from .file_utils import sanitize_filename
+from .file_utils.io import _get_non_conflicting_path, write_file
 
 logger = logging.getLogger(__name__)
-
 _DEFAULT_CHUNK_SIZE = 8192  # 8KB per chunk for streaming downloads
 
 
-def http_get_with_retry(
-    url: str,
-    *,
-    retries: int = 3,
-    timeout: int = 10,
-    backoff: float = 0.5,
-    headers: dict[str, str] | None = None,
-    stream: bool = False,
-) -> requests.Response | None:
+def _normalize_url(url: str) -> str:
     """
-    Perform a GET request with retry support.
-
-    :param url: URL to request.
-    :param retries: Number of retry attempts.
-    :param timeout: Timeout in seconds per request.
-    :param backoff: Base backoff delay between retries.
-    :param headers: Optional HTTP headers.
-    :param stream: Whether to stream the response.
-    :return: Response object if successful, else None.
+    Ensure URL has scheme, defaulting to https:// if missing.
     """
-    for attempt in range(1, retries + 1):
-        try:
-            response = requests.get(
-                url, timeout=timeout, headers=headers, stream=stream
-            )
-            response.raise_for_status()
-            return response
-        except requests.RequestException as e:
-            logger.warning("[http] Attempt %s/%s failed: %s", attempt, retries, e)
-            if attempt < retries:
-                sleep_time = backoff * (2 ** (attempt - 1)) + random.uniform(0, 0.1)
-                time.sleep(sleep_time)
-        except Exception as e:
-            logger.error("[http] Unexpected error: %s", e)
-            break
-
-    logger.error("[http] Failed after %s attempts: %s", retries, url)
-    return None
-
+    if url.startswith("//"):
+        return "https:" + url
+    if not url.startswith(("http://", "https://")):
+        return "https://" + url
+    return url
 
-def image_url_to_filename(url: str) -> str:
-    """
-    Parse and sanitize a image filename from a URL.
-    If no filename or suffix exists, fallback to default name and extension.
 
-    :param url: URL string
-    :return: Safe filename string
-    """
+def _build_filepath(
+    folder: Path,
+    url: str,
+    filename: str | None,
+    default_suffix: str,
+    on_exist: Literal["overwrite", "skip", "rename"],
+) -> Path:
     parsed_url = urlparse(url)
-    path = unquote(parsed_url.path)
-    filename = Path(path).name
-
-    if not filename:
-        filename = "image"
-
-    if not Path(filename).suffix:
-        filename += DEFAULT_IMAGE_SUFFIX
-
-    return filename
+    url_path = Path(unquote(parsed_url.path))
+
+    raw_name = filename or url_path.name or "unnamed"
+    name = sanitize_filename(raw_name)
+    suffix = default_suffix or url_path.suffix
+    if suffix and not suffix.startswith("."):
+        suffix = "." + suffix
+
+    file_path = folder / name
+    if not file_path.suffix and suffix:
+        file_path = file_path.with_suffix(suffix)
+
+    if on_exist == "rename":
+        file_path = _get_non_conflicting_path(file_path)
+    return file_path
+
+
+def _make_session(
+    retries: int,
+    backoff: float,
+    headers: dict[str, str] | None,
+) -> requests.Session:
+    session = requests.Session()
+    session.headers.update(headers or DEFAULT_HEADERS)
+
+    retry = Retry(
+        total=retries,
+        backoff_factor=backoff,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods={"GET", "HEAD", "OPTIONS"},
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
 
 
-def download_image(
+def download(
     url: str,
-    target_folder: str | Path | None = None,
-    target_name: str | None = None,
+    target_dir: str | Path | None = None,
+    filename: str | None = None,
     *,
     timeout: int = 10,
     retries: int = 3,
     backoff: float = 0.5,
     headers: dict[str, str] | None = None,
+    stream: bool = False,
     on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
+    default_suffix: str = "",
+    chunk_size: int = _DEFAULT_CHUNK_SIZE,
 ) -> Path | None:
     """
-    Download an image from `url` and save it to `target_folder`, returning the Path.
-    Can override the filename via `target_name`.
-
-    :param url: Image URL. Can start with 'http', '//', or without protocol.
-    :param target_folder: Directory to save into (defaults to cwd).
-    :param target_name: Optional filename (with or without extension).
-    :param timeout: Request timeout in seconds.
-    :param retries: Number of retry attempts.
-    :param backoff: Base delay between retries (exponential backoff).
-    :param on_exist: What to do if file exists: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved image, or `None` on any failure.
+    Download a URL to disk, with retries, optional rename/skip, and cleanup on failure.
+
+    :param url: the file URL.
+    :param target_dir: directory to save into.
+    :param filename: override the basename (else from URL path).
+    :param timeout: per-request timeout.
+    :param retries: GET retry count.
+    :param backoff: exponential backoff base.
+    :param headers: optional headers.
+    :param stream: Whether to stream the response.
+    :param on_exist: if 'skip', return filepath; if 'rename', auto-rename.
+    :param default_suffix: used if no suffix in URL or filename.
+    :param chunk_size: streaming chunk size.
+    :return: path to the downloaded file.
     """
-    # Normalize URL
-    if url.startswith("//"):
-        url = "https:" + url
-    elif not url.startswith("http"):
-        url = "https://" + url
+    url = _normalize_url(url)
 
-    folder = Path(target_folder) if target_folder else Path.cwd()
+    folder = Path(target_dir) if target_dir else Path.cwd()
     folder.mkdir(parents=True, exist_ok=True)
 
-    if target_name:
-        name = target_name
-        if not Path(name).suffix:
-            # infer ext from URL-derived name
-            name += Path(image_url_to_filename(url)).suffix
-    else:
-        name = image_url_to_filename(url)
-    save_path = folder / name
-
-    # Handle existing file
-    if save_path.exists():
-        if on_exist == "skip":
-            logger.debug("Skipping download; file exists: %s", save_path)
-            return save_path
-        if on_exist == "rename":
-            save_path = _get_non_conflicting_path(save_path)
-
-    # Proceed with download
-    resp = http_get_with_retry(
-        url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=headers or DEFAULT_HEADERS,
-        stream=False,
-    )
-
-    if not (resp and resp.ok):
-        logger.warning(
-            "Failed to download %s (status=%s)",
-            url,
-            getattr(resp, "status_code", None),
-        )
-        return None
-
-    # Write to disk
-    try:
-        _write_file(
-            content=resp.content,
-            filepath=save_path,
-            mode="wb",
-            on_exist=on_exist,
-        )
-        return save_path
-    except Exception:
-        logger.exception("Error saving image to %s", save_path)
-    return None
-
-
-def download_font_file(
-    url: str,
-    target_folder: str | Path,
-    *,
-    timeout: int = 10,
-    retries: int = 3,
-    backoff: float = 0.5,
-    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
-) -> Path | None:
-    """
-    Download a font file from a URL and save it locally with retry and overwrite control
-
-    :param url: Fully-qualified font file URL.
-    :param target_folder: Local folder to save the font file.
-    :param timeout: Timeout for each request (in seconds).
-    :param retries: Number of retry attempts.
-    :param backoff: Base backoff time between retries (in seconds).
-    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved font file, or None if failed.
-    """
-    # Validate and parse URL
-    parsed = urlparse(url)
-    if not parsed.scheme or not parsed.netloc:
-        logger.warning("[font] Invalid URL: %s", url)
-        return None
-
-    # Determine filename
-    filename = Path(unquote(parsed.path)).name
-    if not filename:
-        logger.warning("[font] Could not extract filename from URL: %s", url)
-        return None
-
-    # Resolve save path
-    target_folder = Path(target_folder)
-    target_folder.mkdir(parents=True, exist_ok=True)
-    font_path = target_folder / filename
-
-    # If skip and file exists -> return immediately
-    if on_exist == "skip" and font_path.exists():
-        logger.debug("[font] File exists, skipping download: %s", font_path)
-        return font_path
-
-    # Retry download with exponential backoff
-    response = http_get_with_retry(
+    save_path = _build_filepath(
+        folder,
         url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=DEFAULT_HEADERS,
-        stream=True,
+        filename,
+        default_suffix,
+        on_exist,
     )
 
-    if response:
-        try:
-            if on_exist == "rename":
-                font_path = _get_non_conflicting_path(font_path)
-
-            with open(font_path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=_DEFAULT_CHUNK_SIZE):
-                    if chunk:
-                        f.write(chunk)
-
-            logger.debug("[font] Font saved to: %s", font_path)
-            return font_path
-
-        except Exception as e:
-            logger.error("[font] Error writing font to disk: %s", e)
-
-    return None
-
-
-def download_js_file(
-    url: str,
-    target_folder: str | Path,
-    *,
-    timeout: int = 10,
-    retries: int = 3,
-    backoff: float = 0.5,
-    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
-) -> Path | None:
-    """
-    Download a JavaScript (.js) file from a URL and save it locally.
-
-    :param url: Fully-qualified JS file URL.
-    :param target_folder: Local folder to save the JS file.
-    :param timeout: Timeout for each request (in seconds).
-    :param retries: Number of retry attempts.
-    :param backoff: Base backoff time between retries (in seconds).
-    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved JS file, or None if failed.
-    """
-    parsed = urlparse(url)
-    if not parsed.scheme or not parsed.netloc:
-        logger.warning("[js] Invalid URL: %s", url)
-        return None
-
-    # Determine filename
-    filename = Path(unquote(parsed.path)).name
-    if not filename.endswith(".js"):
-        filename += ".js"
-
-    target_folder = Path(target_folder)
-    target_folder.mkdir(parents=True, exist_ok=True)
-    save_path = target_folder / filename
-
-    if on_exist == "skip" and save_path.exists():
-        logger.debug("[js] File exists, skipping download: %s", save_path)
+    # Handle existing file
+    if save_path.exists() and on_exist == "skip":
+        logger.debug("Skipping download; file exists: %s", save_path)
         return save_path
 
-    response = http_get_with_retry(
-        url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=DEFAULT_HEADERS,
-        stream=False,
-    )
-
-    if response and response.ok:
-        content = response.content
-
-        if on_exist == "rename":
-            save_path = _get_non_conflicting_path(save_path)
-
+    with _make_session(retries, backoff, headers) as session:
         try:
-            _write_file(content=content, filepath=save_path, mode="wb")
-            logger.debug("[js] JS file saved to: %s", save_path)
-            return save_path
+            resp = session.get(url, timeout=timeout, stream=stream)
+            resp.raise_for_status()
         except Exception as e:
-            logger.error("[js] Error writing JS to disk: %s", e)
-
+            logger.warning("[download] request failed: %s", e)
+            return None
+
+        # Write to disk
+        if stream:
+            try:
+                with open(save_path, "wb") as f:
+                    for chunk in resp.iter_content(chunk_size=chunk_size):
+                        if chunk:
+                            f.write(chunk)
+                return save_path
+            except Exception as e:
+                logger.warning("[download] write failed: %s", e)
+                save_path.unlink(missing_ok=True)
+                return None
+        else:
+            return write_file(
+                content=resp.content,
+                filepath=save_path,
+                write_mode="wb",
+                on_exist=on_exist,
+            )
     return None

novel_downloader/utils/state.py

@@ -2,23 +2,22 @@
 """
 novel_downloader.utils.state
 ----------------------------
-State management for user preferences and runtime flags.
 
-Supported sections:
-- general: global preferences (e.g. language)
-- sites: per-site flags & data (e.g. manual_login, cookies)
+State management for user preferences and runtime flags.
 """
+
+__all__ = ["StateManager", "state_mgr"]
+
 import json
 from pathlib import Path
 from typing import Any
 
-from .constants import STATE_FILE
+from novel_downloader.utils.constants import STATE_FILE
 
 
 class StateManager:
     """
     Manages persistent state for user preferences and runtime flags.
-    Stores data in JSON at STATE_FILE.
     """
 
     def __init__(self, path: Path = STATE_FILE) -> None:
@@ -49,26 +48,6 @@ class StateManager:
         content = json.dumps(self._data, ensure_ascii=False, indent=2)
         self._path.write_text(content, encoding="utf-8")
 
-    def _parse_cookie_string(self, cookie_str: str) -> dict[str, str]:
-        """
-        Parse a Cookie header string into a dict.
-
-        :param cookie_str: e.g. 'k1=v1; k2=v2; k3'
-        :return: mapping cookie names to values (missing '=' yields empty string)
-        :rtype: Dict[str, str]
-        """
-        cookies: dict[str, str] = {}
-        for item in cookie_str.split(";"):
-            item = item.strip()
-            if not item:
-                continue
-            if "=" in item:
-                k, v = item.split("=", 1)
-                cookies[k.strip()] = v.strip()
-            else:
-                cookies[item] = ""
-        return cookies
-
     def get_language(self) -> str:
         """
         Load the user's language preference, defaulting to 'zh'.
@@ -87,69 +66,5 @@ class StateManager:
         self._data.setdefault("general", {})["lang"] = lang
         self._save()
 
-    def get_manual_login_flag(self, site: str) -> bool:
-        """
-        Retrieve the manual login requirement flag for a specific site.
-
-        :param site: Site identifier (e.g. 'qidian', 'bqg')
-        :return: True if manual login is required (defaults to True)
-        """
-        val = self._data.get("sites", {}).get(site, {}).get("manual_login", True)
-        return bool(val)
-
-    def set_manual_login_flag(self, site: str, flag: bool) -> None:
-        """
-        Set the 'manual_login' flag for a specific site.
-
-        :param flag: True if the site requires manual login.
-        :param site: Site identifier (e.g. 'qidian', 'bqg')
-        """
-        sites = self._data.setdefault("sites", {})
-        site_data = sites.setdefault(site, {})
-        site_data["manual_login"] = flag
-        self._save()
-
-    def get_cookies(self, site: str) -> dict[str, str]:
-        """
-        Retrieve the persisted cookies for a specific site.
-
-        :param site: Site identifier (e.g. 'qidian', 'bqg')
-        :return: A dict mapping cookie names to values. Returns empty dict if not set.
-        """
-        cookies = self._data.get("sites", {}).get(site, {}).get("cookies", {})
-        return {str(k): str(v) for k, v in cookies.items()}
-
-    def set_cookies(self, site: str, cookies: str | dict[str, str]) -> None:
-        """
-        Persist (overwrite) the cookies for a specific site.
-
-        :param site: Site identifier (e.g. 'qidian', 'bqg')
-        :param cookies: Either a dict mapping cookie names to values,
-                        or a string (JSON or 'k=v; k2=v2') to be parsed.
-        :raises TypeError: if cookies is neither str nor dict
-        """
-        # 1) normalize to dict
-        if isinstance(cookies, dict):
-            cookies_dict = cookies
-        elif isinstance(cookies, str):
-            # try JSON first
-            try:
-                parsed = json.loads(cookies)
-                if isinstance(parsed, dict):
-                    cookies_dict = parsed  # OK!
-                else:
-                    raise ValueError
-            except Exception:
-                # fallback to "k=v; k2=v2" format
-                cookies_dict = self._parse_cookie_string(cookies)
-        else:
-            raise TypeError("`cookies` must be a dict or a str")
-
-        # 2) persist
-        sites = self._data.setdefault("sites", {})
-        site_data = sites.setdefault(site, {})
-        site_data["cookies"] = {str(k): str(v) for k, v in cookies_dict.items()}
-        self._save()
-
 
 state_mgr = StateManager()

novel_downloader/utils/text_utils/__init__.py

@@ -3,31 +3,26 @@
 novel_downloader.utils.text_utils
 ---------------------------------
 
-Utility modules for text formatting, font mapping, cleaning, and diff display.
-
-Submodules:
-- font_mapping: Replace obfuscated characters using font maps
-- chapter_formatting: Build structured chapter strings from raw content
-- text_cleaning: Remove promo text and check for spam lines
-- diff_display: Generate inline diffs with aligned character markers
+Utility modules for text formatting, cleaning, and diff display.
 """
 
-from .chapter_formatting import format_chapter
-from .diff_display import diff_inline_display
-from .font_mapping import apply_font_mapping
-from .text_cleaning import (
-    clean_chapter_title,
-    content_prefix,
-    is_promotional_line,
-    truncate_half_lines,
-)
-
 __all__ = [
-    "apply_font_mapping",
-    "format_chapter",
-    "clean_chapter_title",
-    "is_promotional_line",
+    "TextCleaner",
+    "get_cleaner",
     "content_prefix",
     "truncate_half_lines",
+    "chinese_to_arabic",
+    "arabic_to_chinese",
     "diff_inline_display",
 ]
+
+from .diff_display import diff_inline_display
+from .numeric_conversion import (
+    arabic_to_chinese,
+    chinese_to_arabic,
+)
+from .text_cleaner import TextCleaner, get_cleaner
+from .truncate_utils import (
+    content_prefix,
+    truncate_half_lines,
+)

novel_downloader/utils/text_utils/diff_display.py

@@ -6,6 +6,8 @@ novel_downloader.utils.text_utils.diff_display
 Generate inline character-level diff between two strings with visual markers.
 """
 
+__all__ = ["diff_inline_display"]
+
 import difflib
 import unicodedata
 
@@ -18,10 +20,10 @@ def _char_width_space(
 
     Fullwidth (F) or Wide (W) characters map to `asian_char`, else `normal_char`.
 
-    :param c:           A single character.
+    :param c: A single character.
     :param normal_char: Replacement for narrow chars (default U+0020).
-    :param asian_char:  Replacement for wide chars (default U+3000).
-    :return:            The appropriate space character.
+    :param asian_char: Replacement for wide chars (default U+3000).
+    :return: The appropriate space character.
     """
     return asian_char if unicodedata.east_asian_width(c) in ("F", "W") else normal_char
 
@@ -33,7 +35,7 @@ def diff_inline_display(old_str: str, new_str: str) -> str:
 
     :param old_str: Original string (prefixed '-' will be trimmed).
     :param new_str: Modified string (prefixed '+' will be trimmed).
-    :return:        A multiline diff display with aligned markers.
+    :return: A multiline diff display with aligned markers.
     """
     space_1 = " "
     space_2 = "\u3000"
@@ -67,8 +69,3 @@ def diff_inline_display(old_str: str, new_str: str) -> str:
             marker_s2 += "".join(_char_width_space(c, mark_1, mark_2) for c in s2_seg)
     output_str = f"-{s1}\n {marker_s1}\n+{s2}\n {marker_s2}"
     return output_str
-
-
-__all__ = [
-    "diff_inline_display",
-]