novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/utils/text_utils/numeric_conversion.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.numeric_conversion
+----------------------------------------------------
+
+Utility functions to convert between Chinese numeral strings
+and Python integers.
+"""
+
+CHINESE_NUMERALS = {
+    "零": 0,
+    "〇": 0,
+    "一": 1,
+    "壹": 1,
+    "二": 2,
+    "两": 2,
+    "贰": 2,
+    "貮": 2,
+    "三": 3,
+    "叁": 3,
+    "四": 4,
+    "肆": 4,
+    "五": 5,
+    "伍": 5,
+    "六": 6,
+    "陆": 6,
+    "七": 7,
+    "柒": 7,
+    "八": 8,
+    "捌": 8,
+    "九": 9,
+    "玖": 9,
+}
+
+CHINESE_UNITS = {
+    "十": 10,
+    "拾": 10,
+    "百": 100,
+    "佰": 100,
+    "千": 1000,
+    "仟": 1000,
+    "万": 10_000,
+    "萬": 10_000,
+    "亿": 100_000_000,
+    "億": 100_000_000,
+    "兆": 10**12,
+    "京": 10**16,
+    "垓": 10**20,
+}
+
+LARGE_UNITS = [
+    ("垓", 10**20),
+    ("京", 10**16),
+    ("兆", 10**12),
+    ("亿", 10**8),
+    ("億", 10**8),
+    ("万", 10**4),
+    ("萬", 10**4),
+]
+
+
+def chinese_to_arabic(s: str) -> int:
+    """
+    Convert a Chinese numeral string into its integer value.
+
+    Examples:
+    ---
+    >>> chinese_to_arabic("一千二百三十四")
+    1234
+    >>> chinese_to_arabic("负一千二百三十四")
+    -1234
+    >>> chinese_to_arabic("一万零三")
+    10003
+    >>> chinese_to_arabic("三亿二千五百")
+    3000002500
+
+    :param s: A string of Chinese numerals, e.g. "三千零二十一", "五亿零七万".
+    :return: The integer value represented by the input string.
+    :raises KeyError: If `s` contains characters not found in the supported
+                      numeral or unit mappings.
+    """
+    if not s:
+        raise ValueError("Input string is empty")
+
+    sign = 1
+    if s[0] in ("负", "-"):
+        sign = -1
+        s = s[1:]
+
+    def _parse_section(sec: str) -> int:
+        """Parse up to 千 unit."""
+        num = 0
+        section_total = 0
+        for ch in sec:
+            if ch in CHINESE_NUMERALS:
+                num = num * 10 + CHINESE_NUMERALS[ch]
+            else:
+                unit = CHINESE_UNITS[ch]
+                section_total += (num or 1) * unit
+                num = 0
+        return section_total + num
+
+    total = 0
+    rest = s
+    for char, val in LARGE_UNITS:
+        if char in rest:
+            left, rest = rest.split(char, 1)
+            total += _parse_section(left) * val
+
+    total += _parse_section(rest)
+
+    return sign * total
+
+
+def arabic_to_chinese(num: int) -> str:
+    """
+    Convert an integer to its Chinese numeral representation.
+
+    Examples:
+    ---
+    >>> arabic_to_chinese(0)
+    "零"
+    >>> arabic_to_chinese(1234)
+    "一千二百三十四"
+    >>> arabic_to_chinese(10003)
+    "一万零三"
+    >>> arabic_to_chinese(-205)
+    "负二百零五"
+    >>> arabic_to_chinese(3000002500)
+    "三十亿零二百五百"  # 3 000 002 500
+
+    :param num: The integer to convert (e.g. 42, -1300).
+    :return: The Chinese-numeral string for `num`.
+    :raises TypeError: If `num` is not an integer.
+    """
+    if not isinstance(num, int):
+        raise TypeError("Input must be an integer.")
+    if num == 0:
+        return "零"
+
+    digits = "零一二三四五六七八九"
+    small_units = ["", "十", "百", "千"]
+    big_units = ["", "万", "亿", "兆", "京", "垓"]
+
+    negative = num < 0
+    num = -num if negative else num
+
+    def _section_to_chinese(sec: int) -> str:
+        """
+        Convert a value 1..9999 into Chinese using 千/百/十 units,
+        without any large unit (万, 亿, ...) or leading '零'.
+        """
+        s = ""
+        unit_pos = 0
+        zero_flag = True
+        while sec > 0:
+            d = sec % 10
+            if d == 0:
+                # only emit one '零' for consecutive zeros
+                if not zero_flag:
+                    s = digits[0] + s
+                    zero_flag = True
+            else:
+                s = digits[d] + small_units[unit_pos] + s
+                zero_flag = False
+            unit_pos += 1
+            sec //= 10
+        return s
+
+    result = ""
+    section_pos = 0
+
+    while num > 0:
+        section = num % 10_000
+        if section != 0:
+            sec_str = _section_to_chinese(section)
+            result = sec_str + big_units[section_pos] + result
+        else:
+            # if there's already something in `result`, and the next non-zero
+            # block will appear further left, we need a '零' separator
+            if result and not result.startswith("零"):
+                result = "零" + result
+
+        num //= 10_000
+        section_pos += 1
+
+    if negative:
+        result = "负" + result
+
+    return result
+
+
+if __name__ == "__main__":
+    import random
+
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    RESET = "\033[0m"
+    random.seed(42)
+
+    fail_count = 0
+    num_list = [
+        ("一千二百三十四", 1234),
+        ("一万五千", 15000),
+        ("一万零三", 10003),
+        ("三亿二千五百", 300002500),
+    ]
+    print("=== chinese_to_arabic() with fixed cases ===")
+    for s, expected in num_list:
+        actual = chinese_to_arabic(s)
+        if actual != expected:
+            print(f"{RED}FAIL:{RESET} “{s}” -> expected {expected}, got {actual}")
+            fail_count += 1
+
+    if fail_count:
+        print(f"{RED}{fail_count} chinese_to_arabic() tests failed.{RESET}\n")
+    else:
+        print(f"{GREEN}All {len(num_list)} chinese_to_arabic() tests passed!{RESET}\n")
+
+    fail_count = 0
+    print("=== Round-trip test for values 0 - 9999 ===")
+    for i in range(10_000):
+        s = arabic_to_chinese(i)
+        r = chinese_to_arabic(s)
+        if r != i:
+            print(f'{RED}FAIL round-trip:{RESET} {i} -> "{s}" -> {r}')
+            fail_count += 1
+            break
+
+    if fail_count:
+        print(f"{RED}{fail_count} round-trip failures in 0 - 9999.{RESET}\n")
+    else:
+        print(f"{GREEN}0 - 9999 round-trip all passed!{RESET}\n")
+
+    fail_count = 0
+    exponents = range(5, 22)  # test around 10^5...
+    print("=== Random round-trip at larger scales ===")
+    for exp in exponents:
+        lower = 10**exp
+        upper = 10 ** (exp + 1)
+        for _ in range(2):
+            i = random.randint(lower, upper - 1)
+            for val in (i, -i):
+                s = arabic_to_chinese(val)
+                r = chinese_to_arabic(s)
+                if r != val:
+                    print(f'{RED}FAIL:{RESET} {val} -> "{s}" -> {r}')
+                    fail_count += 1
+
+    if fail_count:
+        print(f"{RED}{fail_count} random large-scale failures.{RESET}")
+    else:
+        print(f"{GREEN}All random large-scale round-trips passed!{RESET}")

novel_downloader/utils/text_utils/text_cleaner.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.text_cleaner
+----------------------------------------------
+
+Provides utilities to clean novel titles and content
+by removing unwanted patterns, replacing strings.
+"""
+
+import re
+from re import Match, Pattern
+from typing import Protocol, runtime_checkable
+
+from novel_downloader.models import TextCleanerConfig
+
+
+@runtime_checkable
+class Cleaner(Protocol):
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        ...
+
+    def clean_title(self, text: str) -> str:
+        ...
+
+    def clean_content(self, text: str) -> str:
+        ...
+
+
+class NullCleaner(Cleaner):
+    def clean_title(self, text: str) -> str:
+        return text
+
+    def clean_content(self, text: str) -> str:
+        return text
+
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        return text
+
+
+class TextCleaner(Cleaner):
+    """
+    TextCleaner removes invisible characters, strips unwanted patterns,
+    and applies literal replacements in a single pass using a combined regex.
+
+    For regex that never matches, reference:
+
+    https://stackoverflow.com/questions/2930182/regex-to-not-match-anything
+    """
+
+    _INVISIBLE_PATTERN: Pattern[str] = re.compile(r"[\ufeff\u200B\u200C\u200D\u2060]")
+
+    def __init__(self, config: TextCleanerConfig) -> None:
+        """
+        Initialize TextCleaner with the given configuration.
+
+        :param config: TextCleanerConfig instance containing:
+
+            - remove_invisible: whether to strip BOM/zero-width chars
+            - title_remove_patterns: list of regex patterns to delete from titles
+            - content_remove_patterns: list of regex patterns to delete from content
+            - title_replacements: dict of literal replacements for titles
+            - content_replacements: dict of literal replacements for content
+        """
+        self._remove_invisible = config.remove_invisible
+
+        # Build literal‐to‐literal replacement maps
+        self._title_repl_map = config.title_replacements
+        self._content_repl_map = config.content_replacements
+
+        # Deduplicate removal patterns (keep order)
+        title_remove = list(dict.fromkeys(config.title_remove_patterns))
+        content_remove = list(dict.fromkeys(config.content_remove_patterns))
+
+        # Build a single combined regex for title:
+        #   all delete‐patterns OR all escaped replacement‐keys
+        title_parts = title_remove + [re.escape(k) for k in self._title_repl_map]
+        title_parts.sort(
+            key=len, reverse=True
+        )  # longer first to avoid prefix collisions
+        title_pattern = "|".join(title_parts) if title_parts else r"$^"
+        self._title_combined_rx: Pattern[str] = re.compile(title_pattern)
+
+        # Build a single combined regex for content (multiline mode)
+        content_parts = content_remove + [re.escape(k) for k in self._content_repl_map]
+        content_parts.sort(key=len, reverse=True)
+        content_pattern = "|".join(content_parts) if content_parts else r"$^"
+        self._content_combined_rx: Pattern[str] = re.compile(
+            content_pattern, flags=re.MULTILINE
+        )
+
+    def clean_title(self, text: str) -> str:
+        """
+        Clean a title string.
+
+        Steps:
+          1. Optionally strip BOM & zero-width characters.
+          2. Remove unwanted patterns and apply literal replacements in one pass.
+          3. Trim leading/trailing whitespace.
+
+        :param text: Raw title text.
+        :return: Cleaned title.
+        """
+        return self._do_clean(text, self._title_combined_rx, self._title_repl_map)
+
+    def clean_content(self, text: str) -> str:
+        """
+        Clean a content string.
+
+        Steps:
+          1. Optionally strip BOM & zero-width characters.
+          2. Remove unwanted patterns and apply literal replacements in one pass.
+          3. Trim leading/trailing whitespace.
+
+        :param text: Raw content/body text.
+        :return: Cleaned content.
+        """
+        return self._do_clean(text, self._content_combined_rx, self._content_repl_map)
+
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        """
+        Unified clean method to process text as either title or content.
+
+        :param text: Raw text to clean.
+        :param as_title: If True, use title rules; otherwise content rules.
+        :return: Cleaned text.
+        """
+        return self.clean_title(text) if as_title else self.clean_content(text)
+
+    @classmethod
+    def _remove_bom_and_invisible(cls, text: str) -> str:
+        """
+        Remove BOM and zero-width/invisible characters from the text.
+
+        Matches:
+          - U+FEFF (BOM)
+          - U+200B ZERO WIDTH SPACE
+          - U+200C ZERO WIDTH NON-JOINER
+          - U+200D ZERO WIDTH JOINER
+          - U+2060 WORD JOINER
+
+        :param text: Input string possibly containing invisible chars.
+        :return: String with those characters stripped.
+        """
+        return cls._INVISIBLE_PATTERN.sub("", text)
+
+    def _do_clean(
+        self,
+        text: str,
+        combined_rx: Pattern[str],
+        repl_map: dict[str, str],
+    ) -> str:
+        """
+        Core cleaning logic:
+        optional invisible removal, single-pass remove/replace, trimming.
+
+        :param text: Text to clean.
+        :param combined_rx: Compiled regex for removal patterns and replacement keys.
+        :param repl_map: Mapping from matched token to replacement text.
+        :return: Cleaned text.
+        """
+        # Strip invisible chars if configured
+        if self._remove_invisible:
+            text = self._remove_bom_and_invisible(text)
+
+        # Single‐pass removal & replacement
+        def _sub(match: Match[str]) -> str:
+            token = match.group(0)
+            # If token in repl_map -> replacement; else -> delete (empty string)
+            return repl_map.get(token, "")
+
+        text = combined_rx.sub(_sub, text)
+        return text.strip()
+
+
+def get_cleaner(
+    enabled: bool,
+    config: TextCleanerConfig,
+) -> Cleaner:
+    return TextCleaner(config) if enabled else NullCleaner()

novel_downloader/utils/text_utils/truncate_utils.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.truncate_utils
+------------------------------------------------
+
+Tools for truncating text.
+"""
+
+__all__ = [
+    "content_prefix",
+    "truncate_half_lines",
+]
+
+import math
+
+
+def content_prefix(
+    text: str,
+    n: int,
+    ignore_chars: set[str] | None = None,
+) -> str:
+    """
+    Return the prefix of `text` containing the first `n` non-ignored characters.
+
+    :param text: The full input string.
+    :param n: Number of content characters to include.
+    :param ignore_chars: Characters to ignore when counting content.
+    :return: Truncated string preserving original whitespace and line breaks.
+    """
+    ignore = ignore_chars or set()
+    cnt = 0
+
+    for i, ch in enumerate(text):
+        if ch not in ignore:
+            cnt += 1
+            if cnt >= n:
+                return text[: i + 1]
+
+    return text
+
+
+def truncate_half_lines(text: str) -> str:
+    """
+    Keep the first half of the lines (rounded up), preserving line breaks.
+
+    :param text: Full input text
+    :return: Truncated text with first half of lines
+    """
+    lines = text.splitlines()
+    non_empty_lines = [line for line in lines if line.strip()]
+    keep_count = math.ceil(len(non_empty_lines) / 2)
+
+    result_lines = []
+    count = 0
+    for line in lines:
+        result_lines.append(line)
+        if line.strip():
+            count += 1
+        if count >= keep_count:
+            break
+
+    return "\n".join(result_lines)

novel_downloader/utils/time_utils/__init__.py

@@ -4,19 +4,13 @@ novel_downloader.utils.time_utils
 ---------------------------------
 
 Utility functions for time and date-related operations.
-
-Includes:
-- calculate_time_difference:
-    Computes time delta between two timezone-aware datetime strings.
-- sleep_with_random_delay:
-    Sleeps for a random duration, useful for human-like delays or rate limiting.
 """
 
-from .datetime_utils import calculate_time_difference
-from .sleep_utils import async_sleep_with_random_delay, sleep_with_random_delay
-
 __all__ = [
-    "calculate_time_difference",
-    "async_sleep_with_random_delay",
-    "sleep_with_random_delay",
+    "time_diff",
+    "async_jitter_sleep",
+    "jitter_sleep",
 ]
+
+from .datetime_utils import time_diff
+from .sleep_utils import async_jitter_sleep, jitter_sleep

novel_downloader/utils/time_utils/datetime_utils.py

@@ -4,14 +4,12 @@ novel_downloader.utils.time_utils.datetime_utils
 ------------------------------------------------
 
 Time utility functions for timezone-aware date calculations.
-
-Includes:
-- _parse_utc_offset():
-    Converts UTC offset string (e.g. 'UTC+8') to a timezone object.
-- calculate_time_difference():
-    Computes timedelta between two datetime strings, with optional timezones.
 """
 
+__all__ = [
+    "time_diff",
+]
+
 import logging
 import re
 from datetime import UTC, datetime, timedelta, timezone
@@ -49,7 +47,7 @@ def _parse_utc_offset(tz_str: str) -> timezone:
     Parse a timezone string like 'UTC+8' or 'UTC-5' into a datetime.timezone object.
 
     :param tz_str: Timezone in 'UTC±<hours>' format, e.g. 'UTC', 'UTC+8', 'UTC-05'
-    :return:       Corresponding timezone object
+    :return: Corresponding timezone object
     :raises ValueError: if tz_str is not a valid UTC offset format
     """
     tz_str_clean = tz_str.upper().strip()
@@ -70,20 +68,20 @@ def _parse_datetime_flexible(dt_str: str) -> datetime:
     """
     Parse a date/time string in any of several common formats:
 
-      • ISO 8601:             'YYYY-MM-DDTHH:MM:SSZ'
-      • ISO w/ offset:        'YYYY-MM-DDTHH:MM:SS+HH:MM'
-      • 'YYYY-MM-DD HH:MM:SS'
-      • 'YYYY-MM-DD'          (time defaults to 00:00:00)
-      • 'YYYY/MM/DD HH:MM:SS'
-      • 'YYYY/MM/DD HH:MM'
-      • 'YYYY/MM/DD'
-      • 'MM/DD/YYYY HH:MM[:SS] AM/PM'
-      • 'MM/DD/YYYY'
-      • 'DD.MM.YYYY HH:MM'
-      • 'DD.MM.YYYY'
+      * ISO 8601:             'YYYY-MM-DDTHH:MM:SSZ'
+      * ISO w/ offset:        'YYYY-MM-DDTHH:MM:SS+HH:MM'
+      * 'YYYY-MM-DD HH:MM:SS'
+      * 'YYYY-MM-DD'          (time defaults to 00:00:00)
+      * 'YYYY/MM/DD HH:MM:SS'
+      * 'YYYY/MM/DD HH:MM'
+      * 'YYYY/MM/DD'
+      * 'MM/DD/YYYY HH:MM[:SS] AM/PM'
+      * 'MM/DD/YYYY'
+      * 'DD.MM.YYYY HH:MM'
+      * 'DD.MM.YYYY'
 
     :param dt_str: Date/time string to parse.
-    :return:       A naive datetime object.
+    :return: A naive datetime object.
     :raises ValueError: If dt_str does not match the expected formats.
     """
     s = dt_str.strip()
@@ -91,13 +89,10 @@ def _parse_datetime_flexible(dt_str: str) -> datetime:
         if re.fullmatch(pattern, s):
             return datetime.strptime(s, fmt)
 
-    supported = "\n".join(f"  • {fmt}" for _, fmt in _DATETIME_FORMATS)
-    raise ValueError(
-        f"Invalid date/time format: '{dt_str}'\n" f"Supported formats are:\n{supported}"
-    )
+    raise ValueError(f"Invalid date/time format: '{dt_str}'")
 
 
-def calculate_time_difference(
+def time_diff(
     from_time_str: str,
     tz_str: str = "UTC",
     to_time_str: str | None = None,
@@ -107,10 +102,10 @@ def calculate_time_difference(
     Calculate the difference between two datetime values.
 
     :param from_time_str: Date-time string "YYYY-MM-DD HH:MM:SS" for the start.
-    :param tz_str:        Timezone of from_time_str, e.g. 'UTC+8'. Defaults to 'UTC'.
-    :param to_time_str:   Optional date-time string for the end; if None, uses now().
-    :param to_tz_str:     Timezone of to_time_str. Defaults to 'UTC'.
-    :return:              Tuple (days, hours, minutes, seconds).
+    :param tz_str: Timezone of from_time_str, e.g. 'UTC+8'. Defaults to 'UTC'.
+    :param to_time_str: Optional date-time string for the end; if None, uses now().
+    :param to_tz_str: Timezone of to_time_str. Defaults to 'UTC'.
+    :return: Tuple (days, hours, minutes, seconds).
     """
     try:
         # parse start time
@@ -139,8 +134,3 @@ def calculate_time_difference(
     except Exception as e:
         logger.warning("[time] Failed to calculate time difference: %s", e)
         return 999, 23, 59, 59
-
-
-__all__ = [
-    "calculate_time_difference",
-]

novel_downloader/utils/time_utils/sleep_utils.py

@@ -4,12 +4,10 @@ novel_downloader.utils.time_utils.sleep_utils
 ---------------------------------------------
 
 Utilities for adding randomized delays in scripts and bots.
-
-Includes:
-- sleep_with_random_delay(): Sleep between base and base+spread seconds,
-  optionally capped with a max_sleep limit.
 """
 
+__all__ = ["jitter_sleep", "async_jitter_sleep"]
+
 import asyncio
 import logging
 import random
@@ -18,7 +16,7 @@ import time
 logger = logging.getLogger(__name__)
 
 
-def sleep_with_random_delay(
+def jitter_sleep(
     base: float,
     add_spread: float = 0.0,
     mul_spread: float = 1.0,
@@ -39,7 +37,7 @@ def sleep_with_random_delay(
     :param mul_spread: Maximum multiplier factor for base; drawn from [1.0, mul_spread].
     :param max_sleep: Optional upper limit for the final sleep duration.
     """
-    if base < 0 or add_spread < 0 or mul_spread < 0:
+    if base < 0 or add_spread < 0 or mul_spread < 1.0:
         logger.warning(
             "[sleep] Invalid parameters: base=%s, add_spread=%s, mul_spread=%s",
             base,
@@ -61,7 +59,7 @@ def sleep_with_random_delay(
     return
 
 
-async def async_sleep_with_random_delay(
+async def async_jitter_sleep(
     base: float,
     add_spread: float = 0.0,
     mul_spread: float = 1.0,
@@ -100,6 +98,3 @@ async def async_sleep_with_random_delay(
 
     logger.debug("[async time] Sleeping for %.2f seconds", duration)
     await asyncio.sleep(duration)
-
-
-__all__ = ["sleep_with_random_delay", "async_sleep_with_random_delay"]

novel_downloader/web/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web
+--------------------
+
+This module exposes the WEB entry point.
+"""
+
+__all__ = [
+    "web_main",
+]
+
+from .main import web_main

novel_downloader/web/components/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.components
+-------------------------------
+
+Entry point for reusable web UI components
+"""
+
+__all__ = ["navbar"]
+
+from .navigation import navbar