novel-downloader 1.5.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/web/services/cred_broker.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.cred_broker
+-----------------------------------------
+
+In-memory credential request broker
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+from novel_downloader.models import LoginField
+
+from .cred_models import CredRequest
+
+# wait time for credentials before timing out (seconds)
+REQUEST_TIMEOUT: int = 120
+# Per-claim lease time (seconds)
+CLAIM_TTL: int = 15
+
+# Global request store
+_CRED_LOCK = asyncio.Lock()
+_CRED_REQS: dict[str, CredRequest] = {}  # req_id -> CredRequest
+
+
+async def create_cred_request(
+    *,
+    task_id: str,
+    title: str,
+    fields: list[LoginField],
+    prefill: dict[str, str] | None = None,
+) -> CredRequest:
+    """
+    Create and register a new credential request for a task.
+    """
+    async with _CRED_LOCK:
+        req = CredRequest(
+            task_id=task_id,
+            title=title,
+            fields=list(fields),
+            prefill=prefill or {},
+        )
+        _CRED_REQS[req.req_id] = req
+        return req
+
+
+async def claim_next_request(client_id: str) -> CredRequest | None:
+    """
+    Claim the next pending unclaimed request; also releases expired claims.
+    """
+    now = time.monotonic()
+    async with _CRED_LOCK:
+        # release stale claims
+        for r in _CRED_REQS.values():
+            if (
+                (not r.done)
+                and r.claimed_by
+                and r.claimed_at
+                and (now - r.claimed_at) > CLAIM_TTL
+            ):
+                r.claimed_by = None
+                r.claimed_at = None
+        # claim one
+        for r in _CRED_REQS.values():
+            if not r.done and r.claimed_by is None:
+                r.claimed_by = client_id
+                r.claimed_at = now
+                return r
+    return None
+
+
+async def refresh_claim(req_id: str, client_id: str) -> None:
+    """
+    Extend the claim lease for a request if it is still owned by the client.
+    """
+    now = time.monotonic()
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if r and (not r.done) and r.claimed_by == client_id:
+            r.claimed_at = now
+
+
+async def complete_request(req_id: str, result: dict[str, str] | None) -> None:
+    """
+    Resolve a request with credentials (or None for cancel/timeout) and wake waiters.
+    """
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if not r or r.done:
+            return
+        r.result = result
+        r.done = True
+        r.event.set()
+
+
+async def get_req_state(req_id: str) -> tuple[bool, bool]:
+    """
+    Return (exists, done) for a request id.
+    """
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if not r:
+            return False, False
+        return True, r.done
+
+
+def cleanup_request(req_id: str) -> None:
+    """
+    Remove a request from the broker (call after the task consumes the result).
+    """
+    _CRED_REQS.pop(req_id, None)

novel_downloader/web/services/cred_models.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.cred_models
+-----------------------------------------
+
+Lightweight data models for the credential broker
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from uuid import uuid4
+
+from novel_downloader.models import LoginField
+
+
+@dataclass
+class CredRequest:
+    task_id: str
+    title: str
+    fields: list[LoginField]
+    prefill: dict[str, str] = field(default_factory=dict)
+
+    # runtime fields
+    req_id: str = field(default_factory=lambda: uuid4().hex)
+    event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
+    result: dict[str, str] | None = None
+
+    # claim info (times use time.monotonic() seconds)
+    claimed_by: str | None = None
+    claimed_at: float | None = None
+
+    # lifecycle
+    done: bool = False

novel_downloader/web/services/task_manager.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.task_manager
+------------------------------------------
+
+Single-worker FIFO task manager for download jobs
+"""
+
+import asyncio
+import contextlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+from uuid import uuid4
+
+from novel_downloader.config import ConfigAdapter, load_config
+from novel_downloader.core import (
+    get_downloader,
+    get_exporter,
+    get_fetcher,
+    get_parser,
+)
+from novel_downloader.models import (
+    BookConfig,
+    LoginField,
+)
+from novel_downloader.utils.cookies import parse_cookies
+
+from .cred_broker import (
+    REQUEST_TIMEOUT,
+    cleanup_request,
+    complete_request,
+    create_cred_request,
+)
+
+Status = Literal["queued", "running", "completed", "cancelled", "failed"]
+
+
+@dataclass
+class DownloadTask:
+    title: str
+    site: str
+    book_id: str
+
+    # runtime state
+    task_id: str = field(default_factory=lambda: uuid4().hex)
+    status: Status = "queued"
+    chapters_total: int = 0
+    chapters_done: int = 0
+    error: str | None = None
+    exported_paths: dict[str, Path] | None = None
+
+    _cancel_event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
+
+    def progress(self) -> float:
+        if self.chapters_total <= 0:
+            return 0.0
+        return self.chapters_done / self.chapters_total
+
+    def cancel(self) -> None:
+        self._cancel_event.set()
+        self.status = "cancelled"
+
+    def is_cancelled(self) -> bool:
+        return self._cancel_event.is_set()
+
+
+class TaskManager:
+    """
+    A cooperative, single-worker queue that executes download tasks in order.
+    """
+
+    def __init__(self) -> None:
+        self.pending: list[DownloadTask] = []
+        self.running: DownloadTask | None = None
+        self.completed: list[DownloadTask] = []
+        self._new_item = asyncio.Event()
+        self._worker_task: asyncio.Task[None] | None = None
+        self._lock = asyncio.Lock()
+
+        self._settings = load_config()
+
+    # ---------- public API ----------
+    async def add_task(self, *, title: str, site: str, book_id: str) -> DownloadTask:
+        """
+        Enqueue a new task and ensure the worker is running; return the created task.
+        """
+        t = DownloadTask(title=title, site=site, book_id=book_id)
+        async with self._lock:
+            self.pending.append(t)
+            self._new_item.set()
+            if not self._worker_task or self._worker_task.done():
+                self._worker_task = asyncio.create_task(self._worker())
+        return t
+
+    async def cancel_task(self, task_id: str) -> bool:
+        """Cancel a task by id (pending or currently running)"""
+        async with self._lock:
+            # cancel pending
+            for i, t in enumerate(self.pending):
+                if t.task_id == task_id:
+                    t.cancel()
+                    self.completed.insert(0, t)
+                    del self.pending[i]
+                    return True
+            # cancel running
+            if self.running and self.running.task_id == task_id:
+                self.running.cancel()
+                return True
+        return False
+
+    def snapshot(self) -> dict[str, Any]:
+        """
+        Return a shallow copy of the current queue state (running, pending, completed).
+        """
+        return {
+            "running": self.running,
+            "pending": list(self.pending),
+            "completed": list(self.completed),
+        }
+
+    # ---------- internals ----------
+    async def _worker(self) -> None:
+        while True:
+            await self._new_item.wait()
+            self._new_item.clear()
+            while True:
+                async with self._lock:
+                    if self.running is not None:
+                        break
+                    if not self.pending:
+                        break
+                    task = self.pending.pop(0)
+                    self.running = task
+
+                await self._run_task(task)
+
+                async with self._lock:
+                    self.completed.insert(0, task)
+                    self.running = None
+
+    async def _run_task(self, task: DownloadTask) -> None:
+        task.status = "running"
+        try:
+            adapter = ConfigAdapter(config=self._settings, site=task.site)
+            downloader_cfg = adapter.get_downloader_config()
+            fetcher_cfg = adapter.get_fetcher_config()
+            parser_cfg = adapter.get_parser_config()
+            exporter_cfg = adapter.get_exporter_config()
+            login_cfg = adapter.get_login_config()
+
+            parser = get_parser(task.site, parser_cfg)
+            exporter = get_exporter(task.site, exporter_cfg)
+
+            async with get_fetcher(task.site, fetcher_cfg) as fetcher:
+                # login if required
+                if downloader_cfg.login_required and not await fetcher.load_state():
+                    login_data = await self._prompt_login_fields(
+                        task, fetcher.login_fields, login_cfg
+                    )
+                    if not await fetcher.login(**login_data):
+                        task.status = "failed"
+                        task.error = "登录失败或已取消"
+                        return
+                    await fetcher.save_state()
+
+                downloader = get_downloader(
+                    fetcher=fetcher,
+                    parser=parser,
+                    site=task.site,
+                    config=downloader_cfg,
+                )
+
+                async def _progress_hook(done: int, total: int) -> None:
+                    if total and (
+                        task.chapters_total <= 0 or total > task.chapters_total
+                    ):
+                        task.chapters_total = total
+                    task.chapters_done = done
+                    # allow cooperative cancel from UI
+                    if task._cancel_event.is_set():
+                        raise asyncio.CancelledError()
+
+                book_cfg: BookConfig = {"book_id": task.book_id}
+                try:
+                    await downloader.download(
+                        book_cfg,
+                        progress_hook=_progress_hook,
+                        cancel_event=task._cancel_event,
+                    )
+                except asyncio.CancelledError:
+                    task.status = "cancelled"
+                    return
+
+                if task.is_cancelled():
+                    task.status = "cancelled"
+                    return
+
+                task.exported_paths = await asyncio.to_thread(
+                    exporter.export, task.book_id
+                )
+
+                if downloader_cfg.login_required and fetcher.is_logged_in:
+                    await fetcher.save_state()
+
+                task.status = "completed"
+
+        except Exception as e:
+            task.status = "failed"
+            task.error = str(e)
+
+    async def _prompt_login_fields(
+        self,
+        task: DownloadTask,
+        fields: list[LoginField],
+        login_config: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Prompt UI login; supports text/password/cookie fields.
+        """
+
+        prefill = (login_config or {}).copy()
+        req = await create_cred_request(
+            task_id=task.task_id,
+            title=task.title,
+            fields=fields,
+            prefill=prefill,
+        )
+
+        # wait for UI to submit or cancel
+        try:
+            await asyncio.wait_for(req.event.wait(), timeout=REQUEST_TIMEOUT)
+        except TimeoutError:
+            await complete_request(req.req_id, None)
+            cleanup_request(req.req_id)
+            return prefill
+
+        if task.is_cancelled():
+            await complete_request(req.req_id, None)
+            cleanup_request(req.req_id)
+            return prefill
+
+        # merge values: prefill -> UI (UI wins)
+        ui_vals: dict[str, str] = req.result or {}
+        cleanup_request(req.req_id)
+
+        merged: dict[str, Any] = {
+            k: v.strip() for k, v in prefill.items() if isinstance(v, str)
+        }
+        merged.update({k: v.strip() for k, v in ui_vals.items() if isinstance(v, str)})
+
+        # parse cookie fields into dicts
+        for f in fields:
+            if f.type == "cookie":
+                raw = merged.get(f.name, "")
+                if isinstance(raw, str) and raw:
+                    with contextlib.suppress(Exception):
+                        # keep raw string if parsing fails
+                        merged[f.name] = parse_cookies(raw)
+
+        return merged
+
+
+manager = TaskManager()

novel_downloader-2.0.0.dist-info/METADATA

@@ -0,0 +1,171 @@
+Metadata-Version: 2.4
+Name: novel-downloader
+Version: 2.0.0
+Summary: A command-line tool for downloading Chinese web novels from Qidian and similar platforms.
+Author-email: Saudade Z <saudadez217@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Saudade Z
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/saudadez21/novel-downloader
+Project-URL: Source, https://github.com/saudadez21/novel-downloader
+Keywords: novel,web novel,qidian,biquge,ebook
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Chinese (Simplified)
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rich
+Requires-Dist: nicegui
+Requires-Dist: requests
+Requires-Dist: aiohttp
+Requires-Dist: lxml
+Requires-Dist: platformdirs
+Provides-Extra: font-recovery
+Requires-Dist: numpy; extra == "font-recovery"
+Requires-Dist: fonttools; extra == "font-recovery"
+Requires-Dist: pillow; extra == "font-recovery"
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-lxml; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: commitizen; extra == "dev"
+Dynamic: license-file
+
+# novel-downloader
+
+基于 [aiohttp](https://github.com/aio-libs/aiohttp) 的异步小说下载工具 / 库。支持断点续传、广告过滤与 TXT/EPUB 导出, 提供 CLI 与 Web 图形界面。
+
+> 运行要求: **Python 3.11+** (开发环境: Python 3.12)
+
+## 功能特性
+
+* **可恢复下载**: 运行时自动检测本地已完成的部分, 跳过已下载内容
+* **多格式导出**: 合并所有章节为
+  * `TXT`
+  * `EPUB` (可选打包章节插图)
+* **广告/活动过滤**:
+  * [x] 章节标题过滤
+  * [x] 章节正文过滤
+* **可选字体混淆还原**: `decode_font`
+* **双形态使用**: 命令行 (CLI) 与 Web 图形界面 (GUI)
+
+---
+
+## 安装
+
+使用 `pip` 安装稳定版:
+
+```bash
+pip install novel-downloader
+```
+
+启用字体解密功能 (`decode_font`):
+
+```bash
+pip install "novel-downloader[font-recovery]"
+```
+
+> 参见: [安装](https://github.com/saudadez21/novel-downloader/blob/main/docs/1-installation.md)
+
+---
+
+## 快速开始
+
+### 1. 初始化配置
+
+```bash
+# 生成默认配置 ./settings.toml
+novel-cli config init
+```
+
+编辑生成的 `./settings.toml`, 可修改 `request_interval`、`book_ids` 等配置 (参考 [settings.toml 配置说明](https://github.com/saudadez21/novel-downloader/blob/main/docs/3-settings-schema.md))
+
+### 2. 命令行 (CLI)
+
+```bash
+# 执行下载任务 (示例: 书籍 ID 为 123456, 默认站点为起点)
+novel-cli download 123456
+```
+
+* 支持站点见: [支持站点列表](https://github.com/saudadez21/novel-downloader/blob/main/docs/4-supported-sites.md)
+* 更多示例见: [CLI 使用示例](https://github.com/saudadez21/novel-downloader/blob/main/docs/5-cli-usage-examples.md)
+
+### 3. 图形界面 (GUI / Web)
+
+```bash
+# 启动 Web 界面 (基于当前 settings.toml)
+novel-web
+
+# 如需提供局域网/外网访问 (请自行留意安全与网络环境)
+# novel-web --listen public
+```
+
+---
+
+## 从源码安装 (开发版)
+
+体验最新开发功能:
+
+```bash
+git clone https://github.com/saudadez21/novel-downloader.git
+cd novel-downloader
+pip install .
+# 或安装带可选功能:
+# pip install .[font-recovery]
+```
+
+---
+
+## 文档导航
+
+* [安装](https://github.com/saudadez21/novel-downloader/blob/main/docs/1-installation.md)
+* [配置](https://github.com/saudadez21/novel-downloader/blob/main/docs/2-configuration.md)
+* [settings.toml 配置说明](https://github.com/saudadez21/novel-downloader/blob/main/docs/3-settings-schema.md)
+* [支持站点列表](https://github.com/saudadez21/novel-downloader/blob/main/docs/4-supported-sites.md)
+* [CLI 使用示例](https://github.com/saudadez21/novel-downloader/blob/main/docs/5-cli-usage-examples.md)
+* [复制 Cookies](https://github.com/saudadez21/novel-downloader/blob/main/docs/copy-cookies.md)
+* [文件保存](https://github.com/saudadez21/novel-downloader/blob/main/docs/file-saving.md)
+* [模块与接口文档](https://github.com/saudadez21/novel-downloader/blob/main/docs/api/README.md)
+* [TODO](https://github.com/saudadez21/novel-downloader/blob/main/docs/todo.md)
+* [开发](https://github.com/saudadez21/novel-downloader/blob/main/docs/develop.md)
+
+---
+
+## 项目说明
+
+* 本项目仅供学习和研究使用, **不得**用于任何商业或违法用途; 请遵守目标网站的 `robots.txt` 及相关法律法规
+* 由于网站结构可能变化或其他问题, 可能导致无法正常工作, 请按需自行调整代码或寻找其他解决方案
+* 使用本项目造成的任何法律责任由使用者自行承担, 项目作者不承担相关责任