novel-downloader 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

novel_downloader/utils/node_decryptor/decryptor.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.utils.node_decryptor.decryptor
+-----------------------------------------------
+
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+import uuid
+from dataclasses import dataclass
+from importlib.resources.abc import Traversable
+from pathlib import Path
+from typing import Any, Final
+
+from novel_downloader.utils.constants import (
+    EXPR_TO_JSON_SCRIPT_PATH,
+    JS_SCRIPT_DIR,
+    QD_DECRYPT_SCRIPT_PATH,
+    QQ_DECRYPT_SCRIPT_PATH,
+)
+from novel_downloader.utils.network import download
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class RemoteAsset:
+    url: str
+    basename: str
+
+
+@dataclass(frozen=True)
+class LocalScript:
+    src: Traversable
+    dst_name: str
+
+
+@dataclass(frozen=True)
+class ProviderSpec:
+    name: str
+    decrypt_script: LocalScript
+    fock_asset: RemoteAsset | None = None
+    has_binary_fallback: bool = False
+
+
+QD_SPEC: Final[ProviderSpec] = ProviderSpec(
+    name="qd",
+    decrypt_script=LocalScript(
+        src=QD_DECRYPT_SCRIPT_PATH,
+        dst_name="qidian_decrypt_node.js",
+    ),
+    fock_asset=RemoteAsset(
+        basename="4819793b.qeooxh.js",
+        url="https://cococdn.qidian.com/coco/s12062024/4819793b.qeooxh.js",
+    ),
+    has_binary_fallback=True,
+)
+
+QQ_SPEC: Final[ProviderSpec] = ProviderSpec(
+    name="qq",
+    decrypt_script=LocalScript(
+        src=QQ_DECRYPT_SCRIPT_PATH,
+        dst_name="qq_decrypt_node.js",
+    ),
+    fock_asset=RemoteAsset(
+        basename="cefc2a5d.pz1phw.js",
+        url="https://imgservices-1252317822.image.myqcloud.com/coco/s10192022/cefc2a5d.pz1phw.js",
+    ),
+    has_binary_fallback=False,
+)
+
+EVAL_SPEC: Final[LocalScript] = LocalScript(
+    src=EXPR_TO_JSON_SCRIPT_PATH,
+    dst_name="expr_to_json.js",
+)
+
+
+class NodeDecryptor:
+    """
+    Decrypts chapter payloads using Node-backed scripts and/or a binary fallback.
+    """
+
+    def __init__(self) -> None:
+        """
+        Initialise the decryptor environment.
+        """
+        self.script_dir: Path = JS_SCRIPT_DIR
+        self.script_dir.mkdir(parents=True, exist_ok=True)
+
+        self.has_node: bool = shutil.which("node") is not None
+
+        # Prepared commands (None => unavailable)
+        self._qd_script_cmd: list[str] | None = None
+        self._qq_script_cmd: list[str] | None = None
+        self._eval_script_cmd: list[str] | None = None
+
+        self._prepare_eval_environment()
+        self._prepare_provider(QD_SPEC)  # sets _qd_script_cmd
+        self._prepare_provider(QQ_SPEC)  # sets _qq_script_cmd
+
+    def decrypt_qd(
+        self,
+        ciphertext: str,
+        chapter_id: str,
+        fkp: str,
+        fuid: str,
+    ) -> str:
+        """
+        Qidian decrypt. Uses Node if present; otherwise tries a fallback binary.
+
+        :param ciphertext: Base64-encoded encrypted content.
+        :param chapter_id: The chapter's numeric ID.
+        :param fkp: Base64-encoded Fock key param from the page.
+        :param fuid: Fock user ID param from the page.
+        :return: "" if unavailable or inputs are missing.
+        :raises RuntimeError: if the Node.js subprocess exits with a non-zero code.
+        """
+        if not self._qd_script_cmd:
+            logger.warning("QD decryptor unavailable (no Node and no fallback).")
+            return ""
+        if not (ciphertext and chapter_id and fkp and fuid):
+            logger.debug("QD decrypt: missing required inputs.")
+            return ""
+
+        task_id = uuid.uuid4().hex
+        input_path = self.script_dir / f"qd_in_{task_id}.json"
+        output_path = self.script_dir / f"qd_out_{task_id}.txt"
+
+        try:
+            input_path.write_text(
+                json.dumps([ciphertext, chapter_id, fkp, fuid]),
+                encoding="utf-8",
+            )
+
+            cmd = self._qd_script_cmd + [input_path.name, output_path.name]
+            logger.debug("Running QD decrypt: %s (cwd=%s)", cmd, self.script_dir)
+            proc = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=self.script_dir,
+            )
+
+            if proc.returncode != 0:
+                stderr = (proc.stderr or "").strip()
+                raise RuntimeError(
+                    f"QD decrypt failed: {stderr or 'non-zero exit code'}"
+                )
+
+            return output_path.read_text(encoding="utf-8").strip()
+        finally:
+            input_path.unlink(missing_ok=True)
+            output_path.unlink(missing_ok=True)
+
+    def decrypt_qq(
+        self,
+        ciphertext: str,
+        chapter_id: str,
+        fkp: str,
+        fuid: str,
+    ) -> str:
+        """
+        QQ decrypt. Node-only.
+
+        :param ciphertext: Base64-encoded encrypted content.
+        :param chapter_id: The chapter's numeric ID.
+        :param fkp: Base64-encoded Fock key param from the page.
+        :param fuid: Fock user ID param from the page.
+        :return: "" if Node/script not available or inputs missing.
+        :raises RuntimeError: if the Node.js subprocess exits with a non-zero code.
+        """
+        if not self._qq_script_cmd:
+            logger.info(
+                "QQ decrypt skipped: Node not available or script not prepared."
+            )
+            return ""
+        if not (ciphertext and chapter_id and fkp and fuid):
+            logger.debug("QQ decrypt: missing required inputs.")
+            return ""
+
+        task_id = uuid.uuid4().hex
+        input_path = self.script_dir / f"qq_in_{task_id}.json"
+        output_path = self.script_dir / f"qq_out_{task_id}.txt"
+
+        try:
+            input_path.write_text(
+                json.dumps([ciphertext, chapter_id, fkp, fuid]),
+                encoding="utf-8",
+            )
+
+            cmd = self._qq_script_cmd + [input_path.name, output_path.name]
+            logger.debug("Running QQ decrypt: %s (cwd=%s)", cmd, self.script_dir)
+            proc = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=self.script_dir,
+            )
+
+            if proc.returncode != 0:
+                stderr = (proc.stderr or "").strip()
+                raise RuntimeError(
+                    f"QQ decrypt failed: {stderr or 'non-zero exit code'}"
+                )
+
+            return output_path.read_text(encoding="utf-8").strip()
+        finally:
+            input_path.unlink(missing_ok=True)
+            output_path.unlink(missing_ok=True)
+
+    def eval_to_json(self, js_code: str) -> dict[str, Any]:
+        """
+        Evaluate JS and parse JSON result. Node-only.
+
+        :return: {} if unavailable or input empty.
+        :raises RuntimeError: if the invoked process fails or outputs invalid JSON.
+        """
+        if not self._eval_script_cmd:
+            logger.info(
+                "eval_to_json skipped: Node not available or script not prepared."
+            )
+            return {}
+        if not js_code:
+            logger.debug("eval_to_json: empty input.")
+            return {}
+
+        logger.debug("Running eval_to_json (cwd=%s)", self.script_dir)
+        proc = subprocess.run(
+            self._eval_script_cmd,
+            input=js_code,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            cwd=self.script_dir,
+        )
+        if proc.returncode != 0:
+            stderr = (proc.stderr or "").strip()
+            raise RuntimeError(f"eval_to_json failed: {stderr or 'non-zero exit code'}")
+
+        stdout = (proc.stdout or "").strip()
+        if not stdout:
+            return {}
+
+        try:
+            return json.loads(stdout) or {}
+        except json.JSONDecodeError as exc:
+            logger.error("eval_to_json: invalid JSON output: %s", stdout)
+            raise RuntimeError("eval_to_json produced invalid JSON.") from exc
+
+    def _prepare_provider(self, spec: ProviderSpec) -> None:
+        """
+        Prepare a provider:
+          * If Node is available: copy node script and fetch Fock asset
+          * Else if provider allows binary fallback (QD): try local binary
+          * Else: leave command None (feature disabled)
+        """
+        dst_script = self.script_dir / spec.decrypt_script.dst_name
+
+        if self.has_node:
+            # Prepare Node script + assets
+            try:
+                if not dst_script.exists():
+                    self._copy_from_traversable(spec.decrypt_script.src, dst_script)
+
+                if spec.fock_asset:
+                    fock_path = self.script_dir / spec.fock_asset.basename
+                    if not fock_path.exists():
+                        download(spec.fock_asset.url, self.script_dir)
+
+                cmd: list[str] | None = ["node", str(dst_script)]
+                logger.debug("%s decryptor prepared with Node.", spec.name.upper())
+            except Exception as exc:
+                logger.warning("%s Node prep failed: %s", spec.name.upper(), exc)
+                cmd = None
+        else:
+            # No Node available
+            if spec.has_binary_fallback and spec.name == "qd":
+                try:
+                    from .decryptor_fetcher import ensure_qd_decryptor
+
+                    bin_path = ensure_qd_decryptor(self.script_dir)
+                    cmd = [str(bin_path)]
+                    logger.debug(
+                        "QD decryptor prepared with binary fallback at %s.", bin_path
+                    )
+                except Exception as exc:
+                    logger.error("QD binary fallback unavailable: %s", exc)
+                    cmd = None
+            else:
+                logger.info(
+                    "%s decryptor skipped: Node not available.", spec.name.upper()
+                )
+                cmd = None
+
+        if spec.name == "qd":
+            self._qd_script_cmd = cmd
+        else:
+            self._qq_script_cmd = cmd
+
+    def _prepare_eval_environment(self) -> None:
+        """
+        Prepare eval script (Node-only).
+        """
+        if not self.has_node:
+            logger.info("eval_to_json skipped: Node not available.")
+            self._eval_script_cmd = None
+            return
+
+        dst = self.script_dir / EVAL_SPEC.dst_name
+        try:
+            if not dst.exists():
+                self._copy_from_traversable(EVAL_SPEC.src, dst)
+            self._eval_script_cmd = ["node", str(dst)]
+            logger.debug("eval_to_json prepared with Node.")
+        except Exception as exc:
+            logger.warning("eval_to_json prep failed; feature disabled. (%s)", exc)
+            self._eval_script_cmd = None
+
+    @staticmethod
+    def _copy_from_traversable(src: Traversable, dst: Path) -> None:
+        """Copy a packaged resource (Traversable) to a filesystem path."""
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        with src.open("rb") as rf, open(dst, "wb") as wf:
+            shutil.copyfileobj(rf, wf)
+
+
+_DECRYPTOR: NodeDecryptor | None = None
+
+
+def get_decryptor() -> NodeDecryptor:
+    """
+    Return the singleton NodeDecryptor.
+    """
+    global _DECRYPTOR
+    if _DECRYPTOR is None:
+        _DECRYPTOR = NodeDecryptor()
+    return _DECRYPTOR

novel_downloader/{core/parsers/qidian/utils → utils/node_decryptor}/decryptor_fetcher.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.parsers.qidian.utils.decryptor_fetcher
-------------------------------------------------------------
+novel_downloader.utils.node_decryptor.decryptor_fetcher
+-------------------------------------------------------
 
 Download and cache the *qidian-decryptor* executable from the project's
 GitHub releases.
@@ -10,7 +10,7 @@ GitHub releases.
 from __future__ import annotations
 
 __all__ = [
-    "ensure_decryptor",
+    "ensure_qd_decryptor",
     "RELEASE_VERSION",
 ]
 
@@ -41,13 +41,12 @@ PLATFORM_BINARIES: Final[dict[str, str]] = {
 # --------------------------------------------------------------------------- #
 
 
-def ensure_decryptor(dest_root: Path | None = None) -> Path:
+def ensure_qd_decryptor(dest_root: Path | None = None) -> Path:
     """
     Ensure that the decryptor executable matching the current platform and
     :data:`RELEASE_VERSION` exists locally; download it if necessary.
 
     :param dest_root: Root directory used to cache the binary.
-                      If *None*, the global constant ``JS_SCRIPT_DIR`` is used.
     :return: Path to the ready-to-use executable (inside the version sub-folder).
     :raises RuntimeError: If the current platform is unsupported.
     :raises ValueError: If the downloaded file fails SHA-256 verification.
@@ -56,7 +55,7 @@ def ensure_decryptor(dest_root: Path | None = None) -> Path:
     platform_key = _get_platform_key()
 
     bin_name = PLATFORM_BINARIES[platform_key]
-    # 版本: /<version>/<binary>
+    # /<version>/<binary>
     version_dir = dest_root / RELEASE_VERSION.lstrip("v")
     dest_path = version_dir / bin_name
 

novel_downloader/utils/text_utils/text_cleaner.py

@@ -42,7 +42,7 @@ class TextCleaner(Cleaner):
     TextCleaner removes invisible characters, strips unwanted patterns,
     and applies literal replacements in a single pass using a combined regex.
 
-    For regex that never matches, reference:
+    For regex that never matches (r"$^"), reference:
 
     https://stackoverflow.com/questions/2930182/regex-to-not-match-anything
     """
@@ -53,13 +53,14 @@ class TextCleaner(Cleaner):
         """
         Initialize TextCleaner with the given configuration.
 
-        :param config: TextCleanerConfig instance containing:
+        Configuration fields (from ``TextCleanerConfig``):
+          * remove_invisible: whether to strip BOM/zero-width chars
+          * title_remove_patterns: list of regex patterns to delete from titles
+          * content_remove_patterns: list of regex patterns to delete from content
+          * title_replacements: dict of literal replacements for titles
+          * content_replacements: dict of literal replacements for content
 
-            - remove_invisible: whether to strip BOM/zero-width chars
-            - title_remove_patterns: list of regex patterns to delete from titles
-            - content_remove_patterns: list of regex patterns to delete from content
-            - title_replacements: dict of literal replacements for titles
-            - content_replacements: dict of literal replacements for content
+        :param config: A ``TextCleanerConfig`` instance.
         """
         self._remove_invisible = config.remove_invisible
 
@@ -73,20 +74,23 @@ class TextCleaner(Cleaner):
 
         # Build a single combined regex for title:
         #   all delete‐patterns OR all escaped replacement‐keys
-        title_parts = title_remove + [re.escape(k) for k in self._title_repl_map]
-        title_parts.sort(
-            key=len, reverse=True
-        )  # longer first to avoid prefix collisions
-        title_pattern = "|".join(title_parts) if title_parts else r"$^"
-        self._title_combined_rx: Pattern[str] = re.compile(title_pattern)
+        self._title_combined_rx: re.Pattern[str] | None = None
+        if title_remove or self._title_repl_map:
+            title_parts = title_remove + [re.escape(k) for k in self._title_repl_map]
+            # longer first to avoid prefix collisions
+            title_parts.sort(key=len, reverse=True)
+            self._title_combined_rx = re.compile("|".join(title_parts))
 
         # Build a single combined regex for content (multiline mode)
-        content_parts = content_remove + [re.escape(k) for k in self._content_repl_map]
-        content_parts.sort(key=len, reverse=True)
-        content_pattern = "|".join(content_parts) if content_parts else r"$^"
-        self._content_combined_rx: Pattern[str] = re.compile(
-            content_pattern, flags=re.MULTILINE
-        )
+        self._content_combined_rx: re.Pattern[str] | None = None
+        if content_remove or self._content_repl_map:
+            content_parts = content_remove + [
+                re.escape(k) for k in self._content_repl_map
+            ]
+            content_parts.sort(key=len, reverse=True)
+            self._content_combined_rx = re.compile(
+                "|".join(content_parts), flags=re.MULTILINE
+            )
 
     def clean_title(self, text: str) -> str:
         """
@@ -132,11 +136,11 @@ class TextCleaner(Cleaner):
         Remove BOM and zero-width/invisible characters from the text.
 
         Matches:
-          - U+FEFF (BOM)
-          - U+200B ZERO WIDTH SPACE
-          - U+200C ZERO WIDTH NON-JOINER
-          - U+200D ZERO WIDTH JOINER
-          - U+2060 WORD JOINER
+          * U+FEFF (BOM)
+          * U+200B ZERO WIDTH SPACE
+          * U+200C ZERO WIDTH NON-JOINER
+          * U+200D ZERO WIDTH JOINER
+          * U+2060 WORD JOINER
 
         :param text: Input string possibly containing invisible chars.
         :return: String with those characters stripped.
@@ -146,7 +150,7 @@ class TextCleaner(Cleaner):
     def _do_clean(
         self,
         text: str,
-        combined_rx: Pattern[str],
+        combined_rx: Pattern[str] | None,
         repl_map: dict[str, str],
     ) -> str:
         """
@@ -158,17 +162,22 @@ class TextCleaner(Cleaner):
         :param repl_map: Mapping from matched token to replacement text.
         :return: Cleaned text.
         """
+        if not self._remove_invisible and not combined_rx:
+            return text.strip()
+
         # Strip invisible chars if configured
         if self._remove_invisible:
             text = self._remove_bom_and_invisible(text)
 
         # Single‐pass removal & replacement
-        def _sub(match: Match[str]) -> str:
-            token = match.group(0)
-            # If token in repl_map -> replacement; else -> delete (empty string)
-            return repl_map.get(token, "")
+        if combined_rx:
+
+            def _sub(match: Match[str]) -> str:
+                # If token in repl_map -> replacement; else -> delete (empty string)
+                return repl_map.get(match.group(0), "")
+
+            text = combined_rx.sub(_sub, text)
 
-        text = combined_rx.sub(_sub, text)
         return text.strip()
 
 

novel_downloader/utils/text_utils/truncate_utils.py

@@ -11,8 +11,6 @@ __all__ = [
     "truncate_half_lines",
 ]
 
-import math
-
 
 def content_prefix(
     text: str,
@@ -41,22 +39,13 @@ def content_prefix(
 
 def truncate_half_lines(text: str) -> str:
     """
-    Keep the first half of the lines (rounded up), preserving line breaks.
+    Keep the first half of the lines.
 
     :param text: Full input text
     :return: Truncated text with first half of lines
     """
     lines = text.splitlines()
     non_empty_lines = [line for line in lines if line.strip()]
-    keep_count = math.ceil(len(non_empty_lines) / 2)
-
-    result_lines = []
-    count = 0
-    for line in lines:
-        result_lines.append(line)
-        if line.strip():
-            count += 1
-        if count >= keep_count:
-            break
-
+    keep_count = (len(non_empty_lines) + 1) // 2
+    result_lines = non_empty_lines[:keep_count]
     return "\n".join(result_lines)

novel_downloader/utils/time_utils/sleep_utils.py

@@ -16,50 +16,51 @@ import time
 logger = logging.getLogger(__name__)
 
 
-def jitter_sleep(
+def _calc_sleep_duration(
     base: float,
-    add_spread: float = 0.0,
-    mul_spread: float = 1.0,
-    *,
+    add_spread: float,
+    mul_spread: float,
     max_sleep: float | None = None,
-) -> None:
+    *,
+    log_prefix: str = "sleep",
+) -> float | None:
     """
-    Sleep for a random duration by combining multiplicative and additive jitter.
-
-    The total sleep time is computed as:
-
-        duration = base * uniform(1.0, mul_spread) + uniform(0, add_spread)
+    Compute the jittered sleep duration (in seconds) or return None if params invalid.
 
-    If `max_sleep` is provided, the duration will be capped at that value.
+      duration = base * uniform(1.0, mul_spread) + uniform(0, add_spread)
 
-    :param base: Base sleep time in seconds. Must be >= 0.
-    :param add_spread: Maximum extra seconds to add after scaling base.
-    :param mul_spread: Maximum multiplier factor for base; drawn from [1.0, mul_spread].
-    :param max_sleep: Optional upper limit for the final sleep duration.
+    then optionally capped by max_sleep.
     """
     if base < 0 or add_spread < 0 or mul_spread < 1.0:
         logger.warning(
-            "[sleep] Invalid parameters: base=%s, add_spread=%s, mul_spread=%s",
+            "[%s] Invalid parameters: base=%s, add_spread=%s, mul_spread=%s",
+            log_prefix,
             base,
             add_spread,
             mul_spread,
         )
-        return
+        return None
 
-    # Calculate the raw duration
     multiplicative_jitter = random.uniform(1.0, mul_spread)
-    additive_jitter = random.uniform(0, add_spread)
+    additive_jitter = random.uniform(0.0, add_spread)
     duration = base * multiplicative_jitter + additive_jitter
 
     if max_sleep is not None:
         duration = min(duration, max_sleep)
 
-    logger.debug("[time] Sleeping for %.2f seconds", duration)
-    time.sleep(duration)
-    return
+    logger.debug(
+        "[%s] base=%.3f mul=%.3f add=%.3f max=%s -> duration=%.3f",
+        log_prefix,
+        base,
+        multiplicative_jitter,
+        additive_jitter,
+        max_sleep,
+        duration,
+    )
+    return duration
 
 
-async def async_jitter_sleep(
+def jitter_sleep(
     base: float,
     add_spread: float = 0.0,
     mul_spread: float = 1.0,
@@ -67,34 +68,43 @@ async def async_jitter_sleep(
     max_sleep: float | None = None,
 ) -> None:
     """
-    Async sleep for a random duration by combining multiplicative and additive jitter.
-
-    The total sleep time is computed as:
-
-        duration = base * uniform(1.0, mul_spread) + uniform(0, add_spread)
-
-    If `max_sleep` is provided, the duration will be capped at that value.
+    Sleep for a random duration by combining multiplicative and additive jitter.
 
     :param base: Base sleep time in seconds. Must be >= 0.
     :param add_spread: Maximum extra seconds to add after scaling base.
     :param mul_spread: Maximum multiplier factor for base; drawn from [1.0, mul_spread].
     :param max_sleep: Optional upper limit for the final sleep duration.
     """
-    if base < 0 or add_spread < 0 or mul_spread < 1.0:
-        logger.warning(
-            "[async sleep] Invalid parameters: base=%s, add_spread=%s, mul_spread=%s",
-            base,
-            add_spread,
-            mul_spread,
-        )
+    duration = _calc_sleep_duration(
+        base,
+        add_spread,
+        mul_spread,
+        max_sleep,
+        log_prefix="sleep",
+    )
+    if duration is None:
         return
+    time.sleep(duration)
 
-    multiplicative_jitter = random.uniform(1.0, mul_spread)
-    additive_jitter = random.uniform(0, add_spread)
-    duration = base * multiplicative_jitter + additive_jitter
 
-    if max_sleep is not None:
-        duration = min(duration, max_sleep)
+async def async_jitter_sleep(
+    base: float,
+    add_spread: float = 0.0,
+    mul_spread: float = 1.0,
+    *,
+    max_sleep: float | None = None,
+) -> None:
+    """
+    Async sleep for a random duration by combining multiplicative and additive jitter.
 
-    logger.debug("[async time] Sleeping for %.2f seconds", duration)
+    :param base: Base sleep time in seconds. Must be >= 0.
+    :param add_spread: Maximum extra seconds to add after scaling base.
+    :param mul_spread: Maximum multiplier factor for base; drawn from [1.0, mul_spread].
+    :param max_sleep: Optional upper limit for the final sleep duration.
+    """
+    duration = _calc_sleep_duration(
+        base, add_spread, mul_spread, max_sleep, log_prefix="async sleep"
+    )
+    if duration is None:
+        return
     await asyncio.sleep(duration)

novel_downloader/web/main.py

@@ -56,7 +56,7 @@ def web_main() -> None:
     host = "127.0.0.1" if args.listen == "local" else "0.0.0.0"
 
     log_level = get_config_value(["general", "debug", "log_level"], "INFO")
-    setup_logging(log_level=log_level)
+    setup_logging(console_level=log_level)
 
     app.on_startup(mount_exports)
     ui.run(host=host, port=args.port, reload=args.reload)