novel-downloader 1.3.0__py3-none-any.whl → 1.3.2__py3-none-any.whl

novel_downloader/__init__.py

@@ -6,7 +6,7 @@ novel_downloader
 Core package for the Novel Downloader project.
 """
 
-__version__ = "1.3.0"
+__version__ = "1.3.2"
 
 __author__ = "Saudade Z"
 __email__ = "saudadez217@gmail.com"

novel_downloader/cli/download.py

@@ -119,7 +119,7 @@ def download_cli(ctx: Context, book_ids: list[str], site: str) -> None:
             config=downloader_cfg,
         )
 
-        for book_id in book_ids:
+        for book_id in valid_book_ids:
             click.echo(t("download_downloading", book_id=book_id, site=site))
             sync_downloader.download_one(book_id)
 

novel_downloader/config/adapter.py

@@ -90,6 +90,8 @@ class ConfigAdapter:
             disable_images=req.get("disable_images", True),
             mute_audio=req.get("mute_audio", True),
             mode=site_cfg.get("mode", "session"),
+            username=site_cfg.get("username", ""),
+            password=site_cfg.get("password", ""),
         )
 
     def get_downloader_config(self) -> DownloaderConfig:
@@ -150,6 +152,7 @@ class ConfigAdapter:
         naming = out.get("naming", {})
         epub_opts = out.get("epub", {})
         return SaverConfig(
+            cache_dir=gen.get("cache_dir", "./novel_cache"),
             raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
             output_dir=gen.get("output_dir", "./downloads"),
             storage_backend=gen.get("storage_backend", "json"),

novel_downloader/config/models.py

@@ -38,6 +38,8 @@ class RequesterConfig:
     mode: ModeType = "session"
     max_connections: int = 10
     max_rps: float | None = None  # Maximum requests per second
+    username: str = ""
+    password: str = ""
 
 
 # === Downloaders ===
@@ -78,6 +80,7 @@ class ParserConfig:
 # === Savers ===
 @dataclass
 class SaverConfig:
+    cache_dir: str = "./novel_cache"
     raw_data_dir: str = "./raw_data"
     output_dir: str = "./downloads"
     storage_backend: StorageBackend = "json"

novel_downloader/core/downloaders/__init__.py

@@ -8,15 +8,37 @@ specific novel platforms.
 
 Each downloader is responsible for orchestrating the full lifecycle
 of retrieving, parsing, and saving novel content for a given source.
+
+Currently supported platforms:
+- biquge (笔趣阁)
+- esjzone (ESJ Zone)
+- qianbi (铅笔小说)
+- qidian (起点中文网)
+- sfacg (SF轻小说)
+- yamibo (百合会)
+- common (通用架构)
 """
 
-from .biquge import BiqugeDownloader
+from .biquge import BiqugeAsyncDownloader, BiqugeDownloader
 from .common import CommonAsyncDownloader, CommonDownloader
+from .esjzone import EsjzoneAsyncDownloader, EsjzoneDownloader
+from .qianbi import QianbiAsyncDownloader, QianbiDownloader
 from .qidian import QidianDownloader
+from .sfacg import SfacgAsyncDownloader, SfacgDownloader
+from .yamibo import YamiboAsyncDownloader, YamiboDownloader
 
 __all__ = [
+    "BiqugeAsyncDownloader",
     "BiqugeDownloader",
     "CommonAsyncDownloader",
     "CommonDownloader",
+    "EsjzoneAsyncDownloader",
+    "EsjzoneDownloader",
+    "QianbiAsyncDownloader",
+    "QianbiDownloader",
     "QidianDownloader",
+    "SfacgAsyncDownloader",
+    "SfacgDownloader",
+    "YamiboAsyncDownloader",
+    "YamiboDownloader",
 ]

novel_downloader/core/downloaders/biquge/__init__.py

@@ -5,8 +5,10 @@ novel_downloader.core.downloaders.biquge
 
 """
 
+from .biquge_async import BiqugeAsyncDownloader
 from .biquge_sync import BiqugeDownloader
 
 __all__ = [
+    "BiqugeAsyncDownloader",
     "BiqugeDownloader",
 ]

novel_downloader/core/downloaders/biquge/biquge_async.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.biquge.biquge_async
+-----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonAsyncDownloader
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+
+
+class BiqugeAsyncDownloader(CommonAsyncDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/biquge/biquge_sync.py

@@ -7,9 +7,11 @@ novel_downloader.core.downloaders.biquge.biquge_sync
 
 from novel_downloader.config.models import DownloaderConfig
 from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces.parser import ParserProtocol
-from novel_downloader.core.interfaces.saver import SaverProtocol
-from novel_downloader.core.interfaces.sync_requester import SyncRequesterProtocol
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    SaverProtocol,
+    SyncRequesterProtocol,
+)
 
 
 class BiqugeDownloader(CommonDownloader):

novel_downloader/core/downloaders/common/common_async.py

@@ -48,7 +48,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
         Perform login
         """
         if self.login_required and not self._is_logged_in:
-            success = await self.requester.login(max_retries=3)
+            success = await self.requester.login()
             if not success:
                 raise RuntimeError("Login failed")
             self._is_logged_in = True
@@ -60,6 +60,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
         :param book_id: The identifier of the book to download.
         """
         assert isinstance(self.requester, AsyncRequesterProtocol)
+        await self.prepare()
 
         TAG = "[AsyncDownloader]"
         wait_time = self.config.request_interval
@@ -95,7 +96,8 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
         if re_fetch:
             info_html = await self.requester.get_book_info(book_id)
             if self.save_html:
-                save_as_txt(info_html, chapters_html_dir / "info.html")
+                for i, html in enumerate(info_html):
+                    save_as_txt(html, chapters_html_dir / f"info_{i}.html")
             book_info = self.parser.parse_book_info(info_html)
             if book_info.get("book_name") != "未找到书名":
                 save_as_json(book_info, info_path)
@@ -114,7 +116,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
 
         # setup queue, semaphore, executor
         semaphore = asyncio.Semaphore(self.download_workers)
-        queue: asyncio.Queue[tuple[str, str]] = asyncio.Queue()
+        queue: asyncio.Queue[tuple[str, list[str]]] = asyncio.Queue()
         save_queue: asyncio.Queue[ChapterDict] = asyncio.Queue()
         loop = asyncio.get_running_loop()
         executor = (

novel_downloader/core/downloaders/common/common_sync.py

@@ -52,6 +52,17 @@ class CommonDownloader(BaseDownloader):
         """
         super().__init__(requester, parser, saver, config, site)
         self._site = site
+        self._is_logged_in = False
+
+    def prepare(self) -> None:
+        """
+        Perform login
+        """
+        if self.login_required and not self._is_logged_in:
+            success = self.requester.login()
+            if not success:
+                raise RuntimeError("Login failed")
+            self._is_logged_in = True
 
     def download_one(self, book_id: str) -> None:
         """
@@ -59,6 +70,8 @@ class CommonDownloader(BaseDownloader):
 
         :param book_id: The identifier of the book to download.
         """
+        self.prepare()
+
         TAG = "[Downloader]"
         save_html = self.config.save_html
         skip_existing = self.config.skip_existing
@@ -96,8 +109,8 @@ class CommonDownloader(BaseDownloader):
         except Exception:
             info_html = self.requester.get_book_info(book_id)
             if save_html:
-                info_html_path = chapters_html_dir / "info.html"
-                save_as_txt(info_html, info_html_path)
+                for i, html in enumerate(info_html):
+                    save_as_txt(html, chapters_html_dir / f"info_{i}.html")
             book_info = self.parser.parse_book_info(info_html)
             if (
                 book_info.get("book_name", "") != "未找到书名"
@@ -138,14 +151,9 @@ class CommonDownloader(BaseDownloader):
                     chap_html = self.requester.get_book_chapter(book_id, cid)
 
                     if save_html:
-                        html_path = chapters_html_dir / f"{cid}.html"
-                        save_as_txt(chap_html, html_path, on_exist="skip")
-                        logger.debug(
-                            "%s Saved raw HTML for chapter %s to %s",
-                            TAG,
-                            cid,
-                            html_path,
-                        )
+                        for i, html in enumerate(chap_html):
+                            html_path = chapters_html_dir / f"{cid}_{i}.html"
+                            save_as_txt(html, html_path, on_exist="skip")
 
                     chap_json = self.parser.parse_chapter(chap_html, cid)
 

novel_downloader/core/downloaders/esjzone/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.esjzone
+-----------------------------------------
+
+"""
+
+from .esjzone_async import EsjzoneAsyncDownloader
+from .esjzone_sync import EsjzoneDownloader
+
+__all__ = [
+    "EsjzoneAsyncDownloader",
+    "EsjzoneDownloader",
+]

novel_downloader/core/downloaders/esjzone/esjzone_async.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.esjzone.esjzone_async
+-------------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonAsyncDownloader
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+
+
+class EsjzoneAsyncDownloader(CommonAsyncDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "esjzone")

novel_downloader/core/downloaders/esjzone/esjzone_sync.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.esjzone.esjzone_sync
+------------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    SaverProtocol,
+    SyncRequesterProtocol,
+)
+
+
+class EsjzoneDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: SyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "esjzone")

novel_downloader/core/downloaders/qianbi/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qianbi
+----------------------------------------
+
+"""
+
+from .qianbi_async import QianbiAsyncDownloader
+from .qianbi_sync import QianbiDownloader
+
+__all__ = [
+    "QianbiAsyncDownloader",
+    "QianbiDownloader",
+]

novel_downloader/core/downloaders/qianbi/qianbi_async.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qianbi.qianbi_async
+-----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonAsyncDownloader
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+
+
+class QianbiAsyncDownloader(CommonAsyncDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "qianbi")

novel_downloader/core/downloaders/qianbi/qianbi_sync.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qianbi.qianbi_sync
+----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    SaverProtocol,
+    SyncRequesterProtocol,
+)
+
+
+class QianbiDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: SyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "qianbi")

novel_downloader/core/downloaders/qidian/qidian_sync.py

@@ -100,9 +100,9 @@ class QidianDownloader(BaseDownloader):
                 raise FileNotFoundError  # trigger re-fetch
         except Exception:
             info_html = self.requester.get_book_info(book_id)
-            if save_html:
+            if save_html and info_html:
                 info_html_path = chapters_html_dir / "info.html"
-                save_as_txt(info_html, info_html_path)
+                save_as_txt(info_html[0], info_html_path)
             book_info = self.parser.parse_book_info(info_html)
             if (
                 book_info.get("book_name", "") != "未找到书名"
@@ -140,6 +140,9 @@ class QidianDownloader(BaseDownloader):
                 chap_title = chap.get("title", "")
                 self.logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
                 chap_html = self.requester.get_book_chapter(book_id, cid)
+                if not chap_html:
+                    continue
+
                 if scroll:
                     self.requester.scroll_page(wait_time * 2)  # type: ignore[attr-defined]
                 else:
@@ -147,7 +150,7 @@ class QidianDownloader(BaseDownloader):
                         wait_time, mul_spread=1.1, max_sleep=wait_time + 2
                     )
 
-                is_encrypted = self.parser.is_encrypted(chap_html)  # type: ignore[attr-defined]
+                is_encrypted = self.parser.is_encrypted(chap_html[0])  # type: ignore[attr-defined]
 
                 if is_encrypted and encrypted_cs.exists(cid) and skip_existing:
                     self.logger.debug(
@@ -157,18 +160,18 @@ class QidianDownloader(BaseDownloader):
                     )
                     continue
 
-                if save_html and not is_vip(chap_html):
+                if save_html and chap_html and not is_vip(chap_html[0]):
                     folder = chapters_html_dir / (
                         "html_encrypted" if is_encrypted else "html_plain"
                     )
                     html_path = folder / f"{cid}.html"
-                    save_as_txt(chap_html, html_path, on_exist="skip")
+                    save_as_txt(chap_html[0], html_path, on_exist="skip")
                     self.logger.debug(
                         "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
                     )
 
                 chap_json = self.parser.parse_chapter(chap_html, cid)
-                if not chap_json:
+                if not chap_json or not chap_json.get("content"):
                     self.logger.warning(
                         "%s Parsed chapter json is empty, skipping: %s (%s)",
                         TAG,

novel_downloader/core/downloaders/sfacg/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.sfacg
+---------------------------------------
+
+"""
+
+from .sfacg_async import SfacgAsyncDownloader
+from .sfacg_sync import SfacgDownloader
+
+__all__ = [
+    "SfacgAsyncDownloader",
+    "SfacgDownloader",
+]

novel_downloader/core/downloaders/sfacg/sfacg_async.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.sfacg.sfacg_async
+---------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonAsyncDownloader
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+
+
+class SfacgAsyncDownloader(CommonAsyncDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/sfacg/sfacg_sync.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.sfacg.sfacg_sync
+--------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    SaverProtocol,
+    SyncRequesterProtocol,
+)
+
+
+class SfacgDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: SyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/yamibo/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.yamibo
+----------------------------------------
+
+"""
+
+from .yamibo_async import YamiboAsyncDownloader
+from .yamibo_sync import YamiboDownloader
+
+__all__ = [
+    "YamiboAsyncDownloader",
+    "YamiboDownloader",
+]

novel_downloader/core/downloaders/yamibo/yamibo_async.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.yamibo.yamibo_async
+-----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonAsyncDownloader
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+
+
+class YamiboAsyncDownloader(CommonAsyncDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "yamibo")

novel_downloader/core/downloaders/yamibo/yamibo_sync.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.yamibo.yamibo_sync
+----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    SaverProtocol,
+    SyncRequesterProtocol,
+)
+
+
+class YamiboDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: SyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "yamibo")

novel_downloader/core/factory/downloader.py

@@ -7,14 +7,24 @@ This module implements a factory function for creating downloader instances
 based on the site name and parser mode specified in the configuration.
 """
 
+from collections.abc import Callable
 from typing import cast
 
 from novel_downloader.config import DownloaderConfig, load_site_rules
 from novel_downloader.core.downloaders import (
+    BiqugeAsyncDownloader,
     BiqugeDownloader,
     CommonAsyncDownloader,
     CommonDownloader,
+    EsjzoneAsyncDownloader,
+    EsjzoneDownloader,
+    QianbiAsyncDownloader,
+    QianbiDownloader,
     QidianDownloader,
+    SfacgAsyncDownloader,
+    SfacgDownloader,
+    YamiboAsyncDownloader,
+    YamiboDownloader,
 )
 from novel_downloader.core.interfaces import (
     AsyncDownloaderProtocol,
@@ -25,12 +35,30 @@ from novel_downloader.core.interfaces import (
     SyncRequesterProtocol,
 )
 
-# _async_site_map = {
-#     # "biquge": ...
-# }
-_sync_site_map = {
-    "qidian": QidianDownloader,
+AsyncDownloaderBuilder = Callable[
+    [AsyncRequesterProtocol, ParserProtocol, SaverProtocol, DownloaderConfig],
+    AsyncDownloaderProtocol,
+]
+
+SyncDownloaderBuilder = Callable[
+    [SyncRequesterProtocol, ParserProtocol, SaverProtocol, DownloaderConfig],
+    SyncDownloaderProtocol,
+]
+
+_async_site_map: dict[str, AsyncDownloaderBuilder] = {
+    "biquge": BiqugeAsyncDownloader,
+    "esjzone": EsjzoneAsyncDownloader,
+    "qianbi": QianbiAsyncDownloader,
+    "sfacg": SfacgAsyncDownloader,
+    "yamibo": YamiboAsyncDownloader,
+}
+_sync_site_map: dict[str, SyncDownloaderBuilder] = {
     "biquge": BiqugeDownloader,
+    "esjzone": EsjzoneDownloader,
+    "qianbi": QianbiDownloader,
+    "qidian": QidianDownloader,
+    "sfacg": SfacgDownloader,
+    "yamibo": YamiboDownloader,
 }
 
 
@@ -61,8 +89,8 @@ def get_async_downloader(
         raise TypeError("Async mode requires an AsyncRequesterProtocol")
 
     # site-specific
-    # if site_key in _async_site_map:
-    #     return _async_site_map[site_key](requester, parser, saver, config)
+    if site_key in _async_site_map:
+        return _async_site_map[site_key](requester, parser, saver, config)
 
     # fallback
     site_rules = load_site_rules()