novel-downloader 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/utils/text_utils/numeric_conversion.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.numeric_conversion
+----------------------------------------------------
+
+Utility functions to convert between Chinese numeral strings
+and Python integers.
+"""
+
+CHINESE_NUMERALS = {
+    "零": 0,
+    "〇": 0,
+    "一": 1,
+    "壹": 1,
+    "二": 2,
+    "两": 2,
+    "贰": 2,
+    "貮": 2,
+    "三": 3,
+    "叁": 3,
+    "四": 4,
+    "肆": 4,
+    "五": 5,
+    "伍": 5,
+    "六": 6,
+    "陆": 6,
+    "七": 7,
+    "柒": 7,
+    "八": 8,
+    "捌": 8,
+    "九": 9,
+    "玖": 9,
+}
+
+CHINESE_UNITS = {
+    "十": 10,
+    "拾": 10,
+    "百": 100,
+    "佰": 100,
+    "千": 1000,
+    "仟": 1000,
+    "万": 10_000,
+    "萬": 10_000,
+    "亿": 100_000_000,
+    "億": 100_000_000,
+    "兆": 10**12,
+    "京": 10**16,
+    "垓": 10**20,
+}
+
+LARGE_UNITS = [
+    ("垓", 10**20),
+    ("京", 10**16),
+    ("兆", 10**12),
+    ("亿", 10**8),
+    ("億", 10**8),
+    ("万", 10**4),
+    ("萬", 10**4),
+]
+
+
+def chinese_to_arabic(s: str) -> int:
+    """
+    Convert a Chinese numeral string into its integer value.
+
+    Examples:
+    ---
+    >>> chinese_to_arabic("一千二百三十四")
+    1234
+    >>> chinese_to_arabic("负一千二百三十四")
+    -1234
+    >>> chinese_to_arabic("一万零三")
+    10003
+    >>> chinese_to_arabic("三亿二千五百")
+    3000002500
+
+    :param s: A string of Chinese numerals, e.g. "三千零二十一", "五亿零七万".
+    :return: The integer value represented by the input string.
+    :raises KeyError: If `s` contains characters not found in the supported
+                      numeral or unit mappings.
+    """
+    if not s:
+        raise ValueError("Input string is empty")
+
+    sign = 1
+    if s[0] in ("负", "-"):
+        sign = -1
+        s = s[1:]
+
+    def _parse_section(sec: str) -> int:
+        """Parse up to 千 unit."""
+        num = 0
+        section_total = 0
+        for ch in sec:
+            if ch in CHINESE_NUMERALS:
+                num = num * 10 + CHINESE_NUMERALS[ch]
+            else:
+                unit = CHINESE_UNITS[ch]
+                section_total += (num or 1) * unit
+                num = 0
+        return section_total + num
+
+    total = 0
+    rest = s
+    for char, val in LARGE_UNITS:
+        if char in rest:
+            left, rest = rest.split(char, 1)
+            total += _parse_section(left) * val
+
+    total += _parse_section(rest)
+
+    return sign * total
+
+
+def arabic_to_chinese(num: int) -> str:
+    """
+    Convert an integer to its Chinese numeral representation.
+
+    Examples:
+    ---
+    >>> arabic_to_chinese(0)
+    "零"
+    >>> arabic_to_chinese(1234)
+    "一千二百三十四"
+    >>> arabic_to_chinese(10003)
+    "一万零三"
+    >>> arabic_to_chinese(-205)
+    "负二百零五"
+    >>> arabic_to_chinese(3000002500)
+    "三十亿零二百五百"  # 3 000 002 500
+
+    :param num: The integer to convert (e.g. 42, -1300).
+    :return: The Chinese-numeral string for `num`.
+    :raises TypeError: If `num` is not an integer.
+    """
+    if not isinstance(num, int):
+        raise TypeError("Input must be an integer.")
+    if num == 0:
+        return "零"
+
+    digits = "零一二三四五六七八九"
+    small_units = ["", "十", "百", "千"]
+    big_units = ["", "万", "亿", "兆", "京", "垓"]
+
+    negative = num < 0
+    num = -num if negative else num
+
+    def _section_to_chinese(sec: int) -> str:
+        """
+        Convert a value 1..9999 into Chinese using 千/百/十 units,
+        without any large unit (万, 亿, ...) or leading '零'.
+        """
+        s = ""
+        unit_pos = 0
+        zero_flag = True
+        while sec > 0:
+            d = sec % 10
+            if d == 0:
+                # only emit one '零' for consecutive zeros
+                if not zero_flag:
+                    s = digits[0] + s
+                    zero_flag = True
+            else:
+                s = digits[d] + small_units[unit_pos] + s
+                zero_flag = False
+            unit_pos += 1
+            sec //= 10
+        return s
+
+    result = ""
+    section_pos = 0
+
+    while num > 0:
+        section = num % 10_000
+        if section != 0:
+            sec_str = _section_to_chinese(section)
+            result = sec_str + big_units[section_pos] + result
+        else:
+            # if there's already something in `result`, and the next non-zero
+            # block will appear further left, we need a '零' separator
+            if result and not result.startswith("零"):
+                result = "零" + result
+
+        num //= 10_000
+        section_pos += 1
+
+    if negative:
+        result = "负" + result
+
+    return result
+
+
+if __name__ == "__main__":
+    import random
+
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    RESET = "\033[0m"
+    random.seed(42)
+
+    fail_count = 0
+    num_list = [
+        ("一千二百三十四", 1234),
+        ("一万五千", 15000),
+        ("一万零三", 10003),
+        ("三亿二千五百", 300002500),
+    ]
+    print("=== chinese_to_arabic() with fixed cases ===")
+    for s, expected in num_list:
+        actual = chinese_to_arabic(s)
+        if actual != expected:
+            print(f"{RED}FAIL:{RESET} “{s}” -> expected {expected}, got {actual}")
+            fail_count += 1
+
+    if fail_count:
+        print(f"{RED}{fail_count} chinese_to_arabic() tests failed.{RESET}\n")
+    else:
+        print(f"{GREEN}All {len(num_list)} chinese_to_arabic() tests passed!{RESET}\n")
+
+    fail_count = 0
+    print("=== Round-trip test for values 0 - 9999 ===")
+    for i in range(10_000):
+        s = arabic_to_chinese(i)
+        r = chinese_to_arabic(s)
+        if r != i:
+            print(f'{RED}FAIL round-trip:{RESET} {i} -> "{s}" -> {r}')
+            fail_count += 1
+            break
+
+    if fail_count:
+        print(f"{RED}{fail_count} round-trip failures in 0 - 9999.{RESET}\n")
+    else:
+        print(f"{GREEN}0 - 9999 round-trip all passed!{RESET}\n")
+
+    fail_count = 0
+    exponents = range(5, 22)  # test around 10^5...
+    print("=== Random round-trip at larger scales ===")
+    for exp in exponents:
+        lower = 10**exp
+        upper = 10 ** (exp + 1)
+        for _ in range(2):
+            i = random.randint(lower, upper - 1)
+            for val in (i, -i):
+                s = arabic_to_chinese(val)
+                r = chinese_to_arabic(s)
+                if r != val:
+                    print(f'{RED}FAIL:{RESET} {val} -> "{s}" -> {r}')
+                    fail_count += 1
+
+    if fail_count:
+        print(f"{RED}{fail_count} random large-scale failures.{RESET}")
+    else:
+        print(f"{GREEN}All random large-scale round-trips passed!{RESET}")

novel_downloader/utils/text_utils/text_cleaner.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.text_cleaner
+----------------------------------------------
+
+Provides utilities to clean novel titles and content
+by removing unwanted patterns, replacing strings.
+"""
+
+import re
+from re import Match, Pattern
+from typing import Protocol, runtime_checkable
+
+from novel_downloader.models import TextCleanerConfig
+
+
+@runtime_checkable
+class Cleaner(Protocol):
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        ...
+
+    def clean_title(self, text: str) -> str:
+        ...
+
+    def clean_content(self, text: str) -> str:
+        ...
+
+
+class NullCleaner(Cleaner):
+    def clean_title(self, text: str) -> str:
+        return text
+
+    def clean_content(self, text: str) -> str:
+        return text
+
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        return text
+
+
+class TextCleaner(Cleaner):
+    """
+    TextCleaner removes invisible characters, strips unwanted patterns,
+    and applies literal replacements in a single pass using a combined regex.
+
+    For regex that never matches, reference:
+
+    https://stackoverflow.com/questions/2930182/regex-to-not-match-anything
+    """
+
+    _INVISIBLE_PATTERN: Pattern[str] = re.compile(r"[\ufeff\u200B\u200C\u200D\u2060]")
+
+    def __init__(self, config: TextCleanerConfig) -> None:
+        """
+        Initialize TextCleaner with the given configuration.
+
+        :param config: TextCleanerConfig instance containing:
+
+            - remove_invisible: whether to strip BOM/zero-width chars
+            - title_remove_patterns: list of regex patterns to delete from titles
+            - content_remove_patterns: list of regex patterns to delete from content
+            - title_replacements: dict of literal replacements for titles
+            - content_replacements: dict of literal replacements for content
+        """
+        self._remove_invisible = config.remove_invisible
+
+        # Build literal‐to‐literal replacement maps
+        self._title_repl_map = config.title_replacements
+        self._content_repl_map = config.content_replacements
+
+        # Deduplicate removal patterns (keep order)
+        title_remove = list(dict.fromkeys(config.title_remove_patterns))
+        content_remove = list(dict.fromkeys(config.content_remove_patterns))
+
+        # Build a single combined regex for title:
+        #   all delete‐patterns OR all escaped replacement‐keys
+        title_parts = title_remove + [re.escape(k) for k in self._title_repl_map]
+        title_parts.sort(
+            key=len, reverse=True
+        )  # longer first to avoid prefix collisions
+        title_pattern = "|".join(title_parts) if title_parts else r"$^"
+        self._title_combined_rx: Pattern[str] = re.compile(title_pattern)
+
+        # Build a single combined regex for content (multiline mode)
+        content_parts = content_remove + [re.escape(k) for k in self._content_repl_map]
+        content_parts.sort(key=len, reverse=True)
+        content_pattern = "|".join(content_parts) if content_parts else r"$^"
+        self._content_combined_rx: Pattern[str] = re.compile(
+            content_pattern, flags=re.MULTILINE
+        )
+
+    def clean_title(self, text: str) -> str:
+        """
+        Clean a title string.
+
+        Steps:
+          1. Optionally strip BOM & zero-width characters.
+          2. Remove unwanted patterns and apply literal replacements in one pass.
+          3. Trim leading/trailing whitespace.
+
+        :param text: Raw title text.
+        :return: Cleaned title.
+        """
+        return self._do_clean(text, self._title_combined_rx, self._title_repl_map)
+
+    def clean_content(self, text: str) -> str:
+        """
+        Clean a content string.
+
+        Steps:
+          1. Optionally strip BOM & zero-width characters.
+          2. Remove unwanted patterns and apply literal replacements in one pass.
+          3. Trim leading/trailing whitespace.
+
+        :param text: Raw content/body text.
+        :return: Cleaned content.
+        """
+        return self._do_clean(text, self._content_combined_rx, self._content_repl_map)
+
+    def clean(self, text: str, *, as_title: bool = False) -> str:
+        """
+        Unified clean method to process text as either title or content.
+
+        :param text: Raw text to clean.
+        :param as_title: If True, use title rules; otherwise content rules.
+        :return: Cleaned text.
+        """
+        return self.clean_title(text) if as_title else self.clean_content(text)
+
+    @classmethod
+    def _remove_bom_and_invisible(cls, text: str) -> str:
+        """
+        Remove BOM and zero-width/invisible characters from the text.
+
+        Matches:
+          - U+FEFF (BOM)
+          - U+200B ZERO WIDTH SPACE
+          - U+200C ZERO WIDTH NON-JOINER
+          - U+200D ZERO WIDTH JOINER
+          - U+2060 WORD JOINER
+
+        :param text: Input string possibly containing invisible chars.
+        :return: String with those characters stripped.
+        """
+        return cls._INVISIBLE_PATTERN.sub("", text)
+
+    def _do_clean(
+        self,
+        text: str,
+        combined_rx: Pattern[str],
+        repl_map: dict[str, str],
+    ) -> str:
+        """
+        Core cleaning logic:
+        optional invisible removal, single-pass remove/replace, trimming.
+
+        :param text: Text to clean.
+        :param combined_rx: Compiled regex for removal patterns and replacement keys.
+        :param repl_map: Mapping from matched token to replacement text.
+        :return: Cleaned text.
+        """
+        # Strip invisible chars if configured
+        if self._remove_invisible:
+            text = self._remove_bom_and_invisible(text)
+
+        # Single‐pass removal & replacement
+        def _sub(match: Match[str]) -> str:
+            token = match.group(0)
+            # If token in repl_map -> replacement; else -> delete (empty string)
+            return repl_map.get(token, "")
+
+        text = combined_rx.sub(_sub, text)
+        return text.strip()
+
+
+def get_cleaner(
+    enabled: bool,
+    config: TextCleanerConfig,
+) -> Cleaner:
+    return TextCleaner(config) if enabled else NullCleaner()

novel_downloader/utils/text_utils/truncate_utils.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.text_utils.truncate_utils
+------------------------------------------------
+
+Tools for truncating text.
+"""
+
+__all__ = [
+    "content_prefix",
+    "truncate_half_lines",
+]
+
+import math
+
+
+def content_prefix(
+    text: str,
+    n: int,
+    ignore_chars: set[str] | None = None,
+) -> str:
+    """
+    Return the prefix of `text` containing the first `n` non-ignored characters.
+
+    :param text: The full input string.
+    :param n: Number of content characters to include.
+    :param ignore_chars: Characters to ignore when counting content.
+    :return: Truncated string preserving original whitespace and line breaks.
+    """
+    ignore = ignore_chars or set()
+    cnt = 0
+
+    for i, ch in enumerate(text):
+        if ch not in ignore:
+            cnt += 1
+            if cnt >= n:
+                return text[: i + 1]
+
+    return text
+
+
+def truncate_half_lines(text: str) -> str:
+    """
+    Keep the first half of the lines (rounded up), preserving line breaks.
+
+    :param text: Full input text
+    :return: Truncated text with first half of lines
+    """
+    lines = text.splitlines()
+    non_empty_lines = [line for line in lines if line.strip()]
+    keep_count = math.ceil(len(non_empty_lines) / 2)
+
+    result_lines = []
+    count = 0
+    for line in lines:
+        result_lines.append(line)
+        if line.strip():
+            count += 1
+        if count >= keep_count:
+            break
+
+    return "\n".join(result_lines)

novel_downloader/utils/time_utils/__init__.py

@@ -12,11 +12,11 @@ Includes:
     Sleeps for a random duration, useful for human-like delays or rate limiting.
 """
 
-from .datetime_utils import calculate_time_difference
-from .sleep_utils import async_sleep_with_random_delay, sleep_with_random_delay
-
 __all__ = [
     "calculate_time_difference",
     "async_sleep_with_random_delay",
     "sleep_with_random_delay",
 ]
+
+from .datetime_utils import calculate_time_difference
+from .sleep_utils import async_sleep_with_random_delay, sleep_with_random_delay

novel_downloader/utils/time_utils/datetime_utils.py

@@ -12,6 +12,10 @@ Includes:
     Computes timedelta between two datetime strings, with optional timezones.
 """
 
+__all__ = [
+    "calculate_time_difference",
+]
+
 import logging
 import re
 from datetime import UTC, datetime, timedelta, timezone
@@ -139,8 +143,3 @@ def calculate_time_difference(
     except Exception as e:
         logger.warning("[time] Failed to calculate time difference: %s", e)
         return 999, 23, 59, 59
-
-
-__all__ = [
-    "calculate_time_difference",
-]

novel_downloader/utils/time_utils/sleep_utils.py

@@ -10,6 +10,8 @@ Includes:
   optionally capped with a max_sleep limit.
 """
 
+__all__ = ["sleep_with_random_delay", "async_sleep_with_random_delay"]
+
 import asyncio
 import logging
 import random
@@ -100,6 +102,3 @@ async def async_sleep_with_random_delay(
 
     logger.debug("[async time] Sleeping for %.2f seconds", duration)
     await asyncio.sleep(duration)
-
-
-__all__ = ["sleep_with_random_delay", "async_sleep_with_random_delay"]

{novel_downloader-1.4.4.dist-info → novel_downloader-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: novel-downloader
-Version: 1.4.4
+Version: 1.5.0
 Summary: A command-line tool for downloading Chinese web novels from Qidian and similar platforms.
 Author-email: Saudade Z <saudadez217@gmail.com>
 License: MIT License
@@ -83,7 +83,7 @@ Dynamic: license-file
   - EPUB (可选包含章节插图)
 - 支持活动广告过滤:
   - [x] 章节标题
-  - [ ] 章节正文
+  - [x] 章节正文
 
 ---