novel-downloader 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/utils/i18n.py

@@ -6,6 +6,8 @@ novel_downloader.utils.i18n
 Multilingual text dictionary and utility for CLI and interactive mode.
 """
 
+__all__ = ["t"]
+
 import json
 from typing import Any
 

novel_downloader/utils/logger.py

@@ -7,6 +7,8 @@ Provides a configurable logging setup for Python applications.
 Log files are rotated daily and named with the given logger name and current date.
 """
 
+__all__ = ["setup_logging"]
+
 import logging
 from datetime import datetime
 from logging.handlers import TimedRotatingFileHandler

novel_downloader/utils/network.py

@@ -6,295 +6,154 @@ novel_downloader.utils.network
 Utilities for handling HTTP requests and downloading remote resources.
 """
 
+__all__ = ["download"]
+
 import logging
-import random
-import time
 from pathlib import Path
 from typing import Literal
 from urllib.parse import unquote, urlparse
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
-from .constants import DEFAULT_HEADERS, DEFAULT_IMAGE_SUFFIX
+from .constants import DEFAULT_HEADERS
+from .file_utils import sanitize_filename
 from .file_utils.io import _get_non_conflicting_path, _write_file
 
 logger = logging.getLogger(__name__)
-
 _DEFAULT_CHUNK_SIZE = 8192  # 8KB per chunk for streaming downloads
 
 
-def http_get_with_retry(
-    url: str,
-    *,
-    retries: int = 3,
-    timeout: int = 10,
-    backoff: float = 0.5,
-    headers: dict[str, str] | None = None,
-    stream: bool = False,
-) -> requests.Response | None:
+def _normalize_url(url: str) -> str:
     """
-    Perform a GET request with retry support.
-
-    :param url: URL to request.
-    :param retries: Number of retry attempts.
-    :param timeout: Timeout in seconds per request.
-    :param backoff: Base backoff delay between retries.
-    :param headers: Optional HTTP headers.
-    :param stream: Whether to stream the response.
-    :return: Response object if successful, else None.
+    Ensure URL has scheme, defaulting to https:// if missing.
     """
-    for attempt in range(1, retries + 1):
-        try:
-            response = requests.get(
-                url, timeout=timeout, headers=headers, stream=stream
-            )
-            response.raise_for_status()
-            return response
-        except requests.RequestException as e:
-            logger.warning("[http] Attempt %s/%s failed: %s", attempt, retries, e)
-            if attempt < retries:
-                sleep_time = backoff * (2 ** (attempt - 1)) + random.uniform(0, 0.1)
-                time.sleep(sleep_time)
-        except Exception as e:
-            logger.error("[http] Unexpected error: %s", e)
-            break
-
-    logger.error("[http] Failed after %s attempts: %s", retries, url)
-    return None
-
+    if url.startswith("//"):
+        return "https:" + url
+    if not url.startswith(("http://", "https://")):
+        return "https://" + url
+    return url
 
-def image_url_to_filename(url: str) -> str:
-    """
-    Parse and sanitize a image filename from a URL.
-    If no filename or suffix exists, fallback to default name and extension.
 
-    :param url: URL string
-    :return: Safe filename string
-    """
+def _build_filepath(
+    folder: Path,
+    url: str,
+    filename: str | None,
+    default_suffix: str,
+    on_exist: Literal["overwrite", "skip", "rename"],
+) -> Path:
     parsed_url = urlparse(url)
-    path = unquote(parsed_url.path)
-    filename = Path(path).name
-
-    if not filename:
-        filename = "image"
-
-    if not Path(filename).suffix:
-        filename += DEFAULT_IMAGE_SUFFIX
-
-    return filename
+    url_path = Path(unquote(parsed_url.path))
+
+    raw_name = filename or url_path.name or "unnamed"
+    name = sanitize_filename(raw_name)
+    suffix = default_suffix or url_path.suffix
+    if suffix and not suffix.startswith("."):
+        suffix = "." + suffix
+
+    file_path = folder / name
+    if not file_path.suffix and suffix:
+        file_path = file_path.with_suffix(suffix)
+
+    if on_exist == "rename":
+        file_path = _get_non_conflicting_path(file_path)
+    return file_path
+
+
+def _make_session(
+    retries: int,
+    backoff: float,
+    headers: dict[str, str] | None,
+) -> requests.Session:
+    session = requests.Session()
+    session.headers.update(headers or DEFAULT_HEADERS)
+
+    retry = Retry(
+        total=retries,
+        backoff_factor=backoff,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods={"GET", "HEAD", "OPTIONS"},
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
 
 
-def download_image(
+def download(
     url: str,
-    target_folder: str | Path | None = None,
-    target_name: str | None = None,
+    target_dir: str | Path | None = None,
+    filename: str | None = None,
     *,
     timeout: int = 10,
     retries: int = 3,
     backoff: float = 0.5,
     headers: dict[str, str] | None = None,
+    stream: bool = False,
     on_exist: Literal["overwrite", "skip", "rename"] = "overwrite",
+    default_suffix: str = "",
+    chunk_size: int = _DEFAULT_CHUNK_SIZE,
 ) -> Path | None:
     """
-    Download an image from `url` and save it to `target_folder`, returning the Path.
-    Can override the filename via `target_name`.
-
-    :param url: Image URL. Can start with 'http', '//', or without protocol.
-    :param target_folder: Directory to save into (defaults to cwd).
-    :param target_name: Optional filename (with or without extension).
-    :param timeout: Request timeout in seconds.
-    :param retries: Number of retry attempts.
-    :param backoff: Base delay between retries (exponential backoff).
-    :param on_exist: What to do if file exists: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved image, or `None` on any failure.
+    Download a URL to disk, with retries, optional rename/skip, and cleanup on failure.
+
+    :param url: the file URL.
+    :param target_dir: directory to save into.
+    :param filename: override the basename (else from URL path).
+    :param timeout: per-request timeout.
+    :param retries: GET retry count.
+    :param backoff: exponential backoff base.
+    :param headers: optional headers.
+    :param stream: Whether to stream the response.
+    :param on_exist: if 'skip', return filepath; if 'rename', auto-rename.
+    :param default_suffix: used if no suffix in URL or filename.
+    :param chunk_size: streaming chunk size.
+    :return: path to the downloaded file.
     """
-    # Normalize URL
-    if url.startswith("//"):
-        url = "https:" + url
-    elif not url.startswith("http"):
-        url = "https://" + url
+    url = _normalize_url(url)
 
-    folder = Path(target_folder) if target_folder else Path.cwd()
+    folder = Path(target_dir) if target_dir else Path.cwd()
     folder.mkdir(parents=True, exist_ok=True)
 
-    if target_name:
-        name = target_name
-        if not Path(name).suffix:
-            # infer ext from URL-derived name
-            name += Path(image_url_to_filename(url)).suffix
-    else:
-        name = image_url_to_filename(url)
-    save_path = folder / name
-
-    # Handle existing file
-    if save_path.exists():
-        if on_exist == "skip":
-            logger.debug("Skipping download; file exists: %s", save_path)
-            return save_path
-        if on_exist == "rename":
-            save_path = _get_non_conflicting_path(save_path)
-
-    # Proceed with download
-    resp = http_get_with_retry(
-        url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=headers or DEFAULT_HEADERS,
-        stream=False,
-    )
-
-    if not (resp and resp.ok):
-        logger.warning(
-            "Failed to download %s (status=%s)",
-            url,
-            getattr(resp, "status_code", None),
-        )
-        return None
-
-    # Write to disk
-    try:
-        _write_file(
-            content=resp.content,
-            filepath=save_path,
-            mode="wb",
-            on_exist=on_exist,
-        )
-        return save_path
-    except Exception:
-        logger.exception("Error saving image to %s", save_path)
-    return None
-
-
-def download_font_file(
-    url: str,
-    target_folder: str | Path,
-    *,
-    timeout: int = 10,
-    retries: int = 3,
-    backoff: float = 0.5,
-    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
-) -> Path | None:
-    """
-    Download a font file from a URL and save it locally with retry and overwrite control
-
-    :param url: Fully-qualified font file URL.
-    :param target_folder: Local folder to save the font file.
-    :param timeout: Timeout for each request (in seconds).
-    :param retries: Number of retry attempts.
-    :param backoff: Base backoff time between retries (in seconds).
-    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved font file, or None if failed.
-    """
-    # Validate and parse URL
-    parsed = urlparse(url)
-    if not parsed.scheme or not parsed.netloc:
-        logger.warning("[font] Invalid URL: %s", url)
-        return None
-
-    # Determine filename
-    filename = Path(unquote(parsed.path)).name
-    if not filename:
-        logger.warning("[font] Could not extract filename from URL: %s", url)
-        return None
-
-    # Resolve save path
-    target_folder = Path(target_folder)
-    target_folder.mkdir(parents=True, exist_ok=True)
-    font_path = target_folder / filename
-
-    # If skip and file exists -> return immediately
-    if on_exist == "skip" and font_path.exists():
-        logger.debug("[font] File exists, skipping download: %s", font_path)
-        return font_path
-
-    # Retry download with exponential backoff
-    response = http_get_with_retry(
+    save_path = _build_filepath(
+        folder,
         url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=DEFAULT_HEADERS,
-        stream=True,
+        filename,
+        default_suffix,
+        on_exist,
     )
 
-    if response:
-        try:
-            if on_exist == "rename":
-                font_path = _get_non_conflicting_path(font_path)
-
-            with open(font_path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=_DEFAULT_CHUNK_SIZE):
-                    if chunk:
-                        f.write(chunk)
-
-            logger.debug("[font] Font saved to: %s", font_path)
-            return font_path
-
-        except Exception as e:
-            logger.error("[font] Error writing font to disk: %s", e)
-
-    return None
-
-
-def download_js_file(
-    url: str,
-    target_folder: str | Path,
-    *,
-    timeout: int = 10,
-    retries: int = 3,
-    backoff: float = 0.5,
-    on_exist: Literal["overwrite", "skip", "rename"] = "skip",
-) -> Path | None:
-    """
-    Download a JavaScript (.js) file from a URL and save it locally.
-
-    :param url: Fully-qualified JS file URL.
-    :param target_folder: Local folder to save the JS file.
-    :param timeout: Timeout for each request (in seconds).
-    :param retries: Number of retry attempts.
-    :param backoff: Base backoff time between retries (in seconds).
-    :param on_exist: File conflict strategy: 'overwrite', 'skip', or 'rename'.
-    :return: Path to the saved JS file, or None if failed.
-    """
-    parsed = urlparse(url)
-    if not parsed.scheme or not parsed.netloc:
-        logger.warning("[js] Invalid URL: %s", url)
-        return None
-
-    # Determine filename
-    filename = Path(unquote(parsed.path)).name
-    if not filename.endswith(".js"):
-        filename += ".js"
-
-    target_folder = Path(target_folder)
-    target_folder.mkdir(parents=True, exist_ok=True)
-    save_path = target_folder / filename
-
-    if on_exist == "skip" and save_path.exists():
-        logger.debug("[js] File exists, skipping download: %s", save_path)
+    # Handle existing file
+    if save_path.exists() and on_exist == "skip":
+        logger.debug("Skipping download; file exists: %s", save_path)
         return save_path
 
-    response = http_get_with_retry(
-        url,
-        retries=retries,
-        timeout=timeout,
-        backoff=backoff,
-        headers=DEFAULT_HEADERS,
-        stream=False,
-    )
-
-    if response and response.ok:
-        content = response.content
-
-        if on_exist == "rename":
-            save_path = _get_non_conflicting_path(save_path)
-
+    with _make_session(retries, backoff, headers) as session:
         try:
-            _write_file(content=content, filepath=save_path, mode="wb")
-            logger.debug("[js] JS file saved to: %s", save_path)
-            return save_path
+            resp = session.get(url, timeout=timeout, stream=stream)
+            resp.raise_for_status()
         except Exception as e:
-            logger.error("[js] Error writing JS to disk: %s", e)
-
+            logger.warning("[download] request failed: %s", e)
+            return None
+
+        # Write to disk
+        if stream:
+            try:
+                with open(save_path, "wb") as f:
+                    for chunk in resp.iter_content(chunk_size=chunk_size):
+                        if chunk:
+                            f.write(chunk)
+                return save_path
+            except Exception as e:
+                logger.warning("[download] write failed: %s", e)
+                save_path.unlink(missing_ok=True)
+                return None
+        else:
+            return _write_file(
+                content=resp.content,
+                filepath=save_path,
+                write_mode="wb",
+                on_exist=on_exist,
+            )
     return None

novel_downloader/utils/state.py

@@ -8,6 +8,7 @@ Supported sections:
 - general: global preferences (e.g. language)
 - sites: per-site flags & data (e.g. manual_login, cookies)
 """
+
 import json
 from pathlib import Path
 from typing import Any

novel_downloader/utils/text_utils/__init__.py

@@ -6,28 +6,29 @@ novel_downloader.utils.text_utils
 Utility modules for text formatting, font mapping, cleaning, and diff display.
 
 Submodules:
-- font_mapping: Replace obfuscated characters using font maps
-- chapter_formatting: Build structured chapter strings from raw content
-- text_cleaning: Remove promo text and check for spam lines
 - diff_display: Generate inline diffs with aligned character markers
+- numeric_conversion: Convert between Chinese and Arabic numerals
+- text_cleaner: Text cleaning and normalization utilities
+- truncate_utils: Text truncation and content prefix generation
 """
 
-from .chapter_formatting import format_chapter
-from .diff_display import diff_inline_display
-from .font_mapping import apply_font_mapping
-from .text_cleaning import (
-    clean_chapter_title,
-    content_prefix,
-    is_promotional_line,
-    truncate_half_lines,
-)
-
 __all__ = [
-    "apply_font_mapping",
-    "format_chapter",
-    "clean_chapter_title",
-    "is_promotional_line",
+    "TextCleaner",
+    "get_cleaner",
     "content_prefix",
     "truncate_half_lines",
+    "chinese_to_arabic",
+    "arabic_to_chinese",
     "diff_inline_display",
 ]
+
+from .diff_display import diff_inline_display
+from .numeric_conversion import (
+    arabic_to_chinese,
+    chinese_to_arabic,
+)
+from .text_cleaner import TextCleaner, get_cleaner
+from .truncate_utils import (
+    content_prefix,
+    truncate_half_lines,
+)

novel_downloader/utils/text_utils/diff_display.py

@@ -6,6 +6,10 @@ novel_downloader.utils.text_utils.diff_display
 Generate inline character-level diff between two strings with visual markers.
 """
 
+__all__ = [
+    "diff_inline_display",
+]
+
 import difflib
 import unicodedata
 
@@ -67,8 +71,3 @@ def diff_inline_display(old_str: str, new_str: str) -> str:
             marker_s2 += "".join(_char_width_space(c, mark_1, mark_2) for c in s2_seg)
     output_str = f"-{s1}\n {marker_s1}\n+{s2}\n {marker_s2}"
     return output_str
-
-
-__all__ = [
-    "diff_inline_display",
-]