novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/web/services/client_dialog.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.client_dialog
+-------------------------------------------
+
+Register a per-page login dialog and a polling timer to claim/handle credential requests
+"""
+
+import asyncio
+import contextlib
+from typing import Any
+
+from nicegui import ui
+
+from novel_downloader.models import LoginField
+
+from .cred_broker import (
+    claim_next_request,
+    complete_request,
+    get_req_state,
+    refresh_claim,
+)
+
+
+def setup_dialog() -> None:
+    """
+    Register the login dialog and a small poller in the current page context.
+    """
+    client_id = ui.context.client.id
+
+    # local state per page instance
+    curr_req_id: str | None = None
+
+    with ui.dialog() as dialog, ui.card().classes("min-w-[360px]"):
+        title_label = ui.label("登录请求").classes("text-base font-medium")
+
+        # dynamic form container
+        form_col = ui.column().classes("w-full gap-2 mt-2")
+
+        # name -> widget
+        inputs: dict[str, Any] = {}
+
+        def _build_form(req_fields: list[LoginField], prefill: dict[str, str]) -> None:
+            """
+            (Re)build inputs inside the dialog's form_col based on LoginField list.
+            """
+            form_col.clear()
+            inputs.clear()
+            with form_col:
+                for idx, f in enumerate(req_fields):
+                    with ui.column().classes("w-full gap-1"):
+                        ui.label(f.label).classes("text-sm font-medium")
+                        if getattr(f, "description", ""):
+                            ui.label(f.description).classes("text-xs text-grey-6")
+
+                        initial = prefill.get(f.name, f.default or "")
+
+                        # choose widget by type
+                        if f.type == "password":
+                            w = (
+                                ui.input(
+                                    f.label,
+                                    password=True,
+                                    value=initial,
+                                    placeholder=f.placeholder or "",
+                                )
+                                .props("dense")
+                                .classes("w-full")
+                            )
+                            w.label = None
+                        elif f.type == "cookie":
+                            # cookie can be long
+                            w = (
+                                ui.textarea(
+                                    f.label,
+                                    value=initial,
+                                    placeholder=f.placeholder or "",
+                                )
+                                .props("dense")
+                                .classes("w-full")
+                            )
+                            w.label = None
+                        else:
+                            # default: text
+                            w = (
+                                ui.input(
+                                    f.label,
+                                    value=initial,
+                                    placeholder=f.placeholder or "",
+                                )
+                                .props("dense")
+                                .classes("w-full")
+                            )
+                            w.label = None
+
+                        # optional niceties
+                        if getattr(f, "required", False):
+                            w.props("required")
+                        if idx == 0:
+                            w.props("autofocus")
+
+                        inputs[f.name] = w
+
+        def on_cancel() -> None:
+            nonlocal curr_req_id
+            if curr_req_id:
+                asyncio.create_task(complete_request(curr_req_id, None))
+            curr_req_id = None
+            dialog.close()
+            ui.notify("已取消登录")
+
+        def on_submit() -> None:
+            nonlocal curr_req_id
+            if not curr_req_id:
+                return
+            # collect values
+            values: dict[str, str] = {}
+            for name, w in inputs.items():
+                values[name] = (w.value or "").strip()
+            # send to broker
+            asyncio.create_task(complete_request(curr_req_id, values))
+            curr_req_id = None
+            dialog.close()
+            ui.notify("提交成功")
+
+        with ui.row().classes("justify-end w-full mt-2"):
+            ui.button("取消", on_click=on_cancel).props("flat color=grey-7")
+            ui.button("提交", on_click=on_submit)
+
+    dialog.props("persistent")
+
+    async def check_and_open() -> None:
+        nonlocal curr_req_id
+        rid = curr_req_id
+        if rid:
+            exists, done = await get_req_state(rid)
+            if (not exists) or done:
+                curr_req_id = None
+                if dialog.visible:
+                    dialog.close()
+            else:
+                await refresh_claim(rid, client_id)
+            return
+
+        req = await claim_next_request(client_id)
+        if not req:
+            return
+        curr_req_id = req.req_id
+        title_label.text = f"'{req.title}' 需要登录信息"
+        _build_form(req.fields, req.prefill)
+        dialog.open()
+
+    ui.timer(0.5, check_and_open)
+
+    # Clean up if this page's websocket disconnects
+    def _on_disconnect() -> None:
+        nonlocal curr_req_id
+        if curr_req_id:
+            asyncio.create_task(complete_request(curr_req_id, None))
+        curr_req_id = None
+
+    with contextlib.suppress(Exception):
+        ui.context.client.on_disconnect(_on_disconnect)
+        # Fallback: CLAIM_TTL will release stale claims anyway

novel_downloader/web/services/cred_broker.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.cred_broker
+-----------------------------------------
+
+In-memory credential request broker
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+from novel_downloader.models import LoginField
+
+from .cred_models import CredRequest
+
+# wait time for credentials before timing out (seconds)
+REQUEST_TIMEOUT: int = 120
+# Per-claim lease time (seconds)
+CLAIM_TTL: int = 15
+
+# Global request store
+_CRED_LOCK = asyncio.Lock()
+_CRED_REQS: dict[str, CredRequest] = {}  # req_id -> CredRequest
+
+
+async def create_cred_request(
+    *,
+    task_id: str,
+    title: str,
+    fields: list[LoginField],
+    prefill: dict[str, str] | None = None,
+) -> CredRequest:
+    """
+    Create and register a new credential request for a task.
+    """
+    async with _CRED_LOCK:
+        req = CredRequest(
+            task_id=task_id,
+            title=title,
+            fields=list(fields),
+            prefill=prefill or {},
+        )
+        _CRED_REQS[req.req_id] = req
+        return req
+
+
+async def claim_next_request(client_id: str) -> CredRequest | None:
+    """
+    Claim the next pending unclaimed request; also releases expired claims.
+    """
+    now = time.monotonic()
+    async with _CRED_LOCK:
+        # release stale claims
+        for r in _CRED_REQS.values():
+            if (
+                (not r.done)
+                and r.claimed_by
+                and r.claimed_at
+                and (now - r.claimed_at) > CLAIM_TTL
+            ):
+                r.claimed_by = None
+                r.claimed_at = None
+        # claim one
+        for r in _CRED_REQS.values():
+            if not r.done and r.claimed_by is None:
+                r.claimed_by = client_id
+                r.claimed_at = now
+                return r
+    return None
+
+
+async def refresh_claim(req_id: str, client_id: str) -> None:
+    """
+    Extend the claim lease for a request if it is still owned by the client.
+    """
+    now = time.monotonic()
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if r and (not r.done) and r.claimed_by == client_id:
+            r.claimed_at = now
+
+
+async def complete_request(req_id: str, result: dict[str, str] | None) -> None:
+    """
+    Resolve a request with credentials (or None for cancel/timeout) and wake waiters.
+    """
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if not r or r.done:
+            return
+        r.result = result
+        r.done = True
+        r.event.set()
+
+
+async def get_req_state(req_id: str) -> tuple[bool, bool]:
+    """
+    Return (exists, done) for a request id.
+    """
+    async with _CRED_LOCK:
+        r = _CRED_REQS.get(req_id)
+        if not r:
+            return False, False
+        return True, r.done
+
+
+def cleanup_request(req_id: str) -> None:
+    """
+    Remove a request from the broker (call after the task consumes the result).
+    """
+    _CRED_REQS.pop(req_id, None)

novel_downloader/web/services/cred_models.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.cred_models
+-----------------------------------------
+
+Lightweight data models for the credential broker
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from uuid import uuid4
+
+from novel_downloader.models import LoginField
+
+
+@dataclass
+class CredRequest:
+    task_id: str
+    title: str
+    fields: list[LoginField]
+    prefill: dict[str, str] = field(default_factory=dict)
+
+    # runtime fields
+    req_id: str = field(default_factory=lambda: uuid4().hex)
+    event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
+    result: dict[str, str] | None = None
+
+    # claim info (times use time.monotonic() seconds)
+    claimed_by: str | None = None
+    claimed_at: float | None = None
+
+    # lifecycle
+    done: bool = False

novel_downloader/web/services/task_manager.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.web.services.task_manager
+------------------------------------------
+
+Single-worker FIFO task manager for download jobs
+"""
+
+import asyncio
+import contextlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+from uuid import uuid4
+
+from novel_downloader.config import ConfigAdapter, load_config
+from novel_downloader.core import (
+    get_downloader,
+    get_exporter,
+    get_fetcher,
+    get_parser,
+)
+from novel_downloader.models import (
+    BookConfig,
+    LoginField,
+)
+from novel_downloader.utils.cookies import parse_cookies
+
+from .cred_broker import (
+    REQUEST_TIMEOUT,
+    cleanup_request,
+    complete_request,
+    create_cred_request,
+)
+
+Status = Literal["queued", "running", "completed", "cancelled", "failed"]
+
+
+@dataclass
+class DownloadTask:
+    title: str
+    site: str
+    book_id: str
+
+    # runtime state
+    task_id: str = field(default_factory=lambda: uuid4().hex)
+    status: Status = "queued"
+    chapters_total: int = 0
+    chapters_done: int = 0
+    error: str | None = None
+    exported_paths: dict[str, Path] | None = None
+
+    _cancel_event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
+
+    def progress(self) -> float:
+        if self.chapters_total <= 0:
+            return 0.0
+        return self.chapters_done / self.chapters_total
+
+    def cancel(self) -> None:
+        self._cancel_event.set()
+        self.status = "cancelled"
+
+    def is_cancelled(self) -> bool:
+        return self._cancel_event.is_set()
+
+
+class TaskManager:
+    """
+    A cooperative, single-worker queue that executes download tasks in order.
+    """
+
+    def __init__(self) -> None:
+        self.pending: list[DownloadTask] = []
+        self.running: DownloadTask | None = None
+        self.completed: list[DownloadTask] = []
+        self._new_item = asyncio.Event()
+        self._worker_task: asyncio.Task[None] | None = None
+        self._lock = asyncio.Lock()
+
+        self._settings = load_config()
+
+    # ---------- public API ----------
+    async def add_task(self, *, title: str, site: str, book_id: str) -> DownloadTask:
+        """
+        Enqueue a new task and ensure the worker is running; return the created task.
+        """
+        t = DownloadTask(title=title, site=site, book_id=book_id)
+        async with self._lock:
+            self.pending.append(t)
+            self._new_item.set()
+            if not self._worker_task or self._worker_task.done():
+                self._worker_task = asyncio.create_task(self._worker())
+        return t
+
+    async def cancel_task(self, task_id: str) -> bool:
+        """Cancel a task by id (pending or currently running)"""
+        async with self._lock:
+            # cancel pending
+            for i, t in enumerate(self.pending):
+                if t.task_id == task_id:
+                    t.cancel()
+                    self.completed.insert(0, t)
+                    del self.pending[i]
+                    return True
+            # cancel running
+            if self.running and self.running.task_id == task_id:
+                self.running.cancel()
+                return True
+        return False
+
+    def snapshot(self) -> dict[str, Any]:
+        """
+        Return a shallow copy of the current queue state (running, pending, completed).
+        """
+        return {
+            "running": self.running,
+            "pending": list(self.pending),
+            "completed": list(self.completed),
+        }
+
+    # ---------- internals ----------
+    async def _worker(self) -> None:
+        while True:
+            await self._new_item.wait()
+            self._new_item.clear()
+            while True:
+                async with self._lock:
+                    if self.running is not None:
+                        break
+                    if not self.pending:
+                        break
+                    task = self.pending.pop(0)
+                    self.running = task
+
+                await self._run_task(task)
+
+                async with self._lock:
+                    self.completed.insert(0, task)
+                    self.running = None
+
+    async def _run_task(self, task: DownloadTask) -> None:
+        task.status = "running"
+        try:
+            adapter = ConfigAdapter(config=self._settings, site=task.site)
+            downloader_cfg = adapter.get_downloader_config()
+            fetcher_cfg = adapter.get_fetcher_config()
+            parser_cfg = adapter.get_parser_config()
+            exporter_cfg = adapter.get_exporter_config()
+            login_cfg = adapter.get_login_config()
+
+            parser = get_parser(task.site, parser_cfg)
+            exporter = get_exporter(task.site, exporter_cfg)
+
+            async with get_fetcher(task.site, fetcher_cfg) as fetcher:
+                # login if required
+                if downloader_cfg.login_required and not await fetcher.load_state():
+                    login_data = await self._prompt_login_fields(
+                        task, fetcher.login_fields, login_cfg
+                    )
+                    if not await fetcher.login(**login_data):
+                        task.status = "failed"
+                        task.error = "登录失败或已取消"
+                        return
+                    await fetcher.save_state()
+
+                downloader = get_downloader(
+                    fetcher=fetcher,
+                    parser=parser,
+                    site=task.site,
+                    config=downloader_cfg,
+                )
+
+                async def _progress_hook(done: int, total: int) -> None:
+                    if total and (
+                        task.chapters_total <= 0 or total > task.chapters_total
+                    ):
+                        task.chapters_total = total
+                    task.chapters_done = done
+                    # allow cooperative cancel from UI
+                    if task._cancel_event.is_set():
+                        raise asyncio.CancelledError()
+
+                book_cfg: BookConfig = {"book_id": task.book_id}
+                try:
+                    await downloader.download(
+                        book_cfg,
+                        progress_hook=_progress_hook,
+                        cancel_event=task._cancel_event,
+                    )
+                except asyncio.CancelledError:
+                    task.status = "cancelled"
+                    return
+
+                if task.is_cancelled():
+                    task.status = "cancelled"
+                    return
+
+                task.exported_paths = await asyncio.to_thread(
+                    exporter.export, task.book_id
+                )
+
+                if downloader_cfg.login_required and fetcher.is_logged_in:
+                    await fetcher.save_state()
+
+                task.status = "completed"
+
+        except Exception as e:
+            task.status = "failed"
+            task.error = str(e)
+
+    async def _prompt_login_fields(
+        self,
+        task: DownloadTask,
+        fields: list[LoginField],
+        login_config: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Prompt UI login; supports text/password/cookie fields.
+        """
+
+        prefill = (login_config or {}).copy()
+        req = await create_cred_request(
+            task_id=task.task_id,
+            title=task.title,
+            fields=fields,
+            prefill=prefill,
+        )
+
+        # wait for UI to submit or cancel
+        try:
+            await asyncio.wait_for(req.event.wait(), timeout=REQUEST_TIMEOUT)
+        except TimeoutError:
+            await complete_request(req.req_id, None)
+            cleanup_request(req.req_id)
+            return prefill
+
+        if task.is_cancelled():
+            await complete_request(req.req_id, None)
+            cleanup_request(req.req_id)
+            return prefill
+
+        # merge values: prefill -> UI (UI wins)
+        ui_vals: dict[str, str] = req.result or {}
+        cleanup_request(req.req_id)
+
+        merged: dict[str, Any] = {
+            k: v.strip() for k, v in prefill.items() if isinstance(v, str)
+        }
+        merged.update({k: v.strip() for k, v in ui_vals.items() if isinstance(v, str)})
+
+        # parse cookie fields into dicts
+        for f in fields:
+            if f.type == "cookie":
+                raw = merged.get(f.name, "")
+                if isinstance(raw, str) and raw:
+                    with contextlib.suppress(Exception):
+                        # keep raw string if parsing fails
+                        merged[f.name] = parse_cookies(raw)
+
+        return merged
+
+
+manager = TaskManager()