novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/{sfacg/main_parser.py → sfacg.py}

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.parsers.sfacg.main_parser
------------------------------------------------
+novel_downloader.core.parsers.sfacg
+-----------------------------------
 
 """
 
@@ -10,18 +10,33 @@ from typing import Any
 from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.models import ChapterDict
-
-
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["sfacg"],
+)
 class SfacgParser(BaseParser):
-    """ """
+    """
+    Parser for sfacg book pages.
+    """
 
     # Book info XPaths
     _BOOK_NAME_XPATH = '//ul[@class="book_info"]//span[@class="book_newtitle"]/text()'
     _AUTHOR_INFO_XPATH = '//ul[@class="book_info"]//span[@class="book_info3"]/text()'
     _UPDATE_TIME_XPATH = '//ul[@class="book_info"]//span[@class="book_info3"]/br/following-sibling::text()'  # noqa: E501
     _COVER_URL_XPATH = '//ul[@class="book_info"]//li/img/@src'
-    _STATUS_XPATH = '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+    # _STATUS_XPATH = '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+    _STATUS_XPATH = (
+        '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+        ' and (contains(., "完结") or contains(., "连载"))]/text()'
+    )
     _SUMMARY_XPATH = '//ul[@class="book_profile"]/li[@class="book_bk_qs1"]/text()'
 
     # Catalog XPaths
@@ -42,54 +57,35 @@ class SfacgParser(BaseParser):
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
-        """
-        Parse a book info page and extract metadata and chapter structure.
-
-        :param html_list: Raw HTML of the book info page.
-        :return: Parsed metadata and chapter structure as a dictionary.
-        """
+    ) -> BookInfoDict | None:
         if len(html_list) < 2:
-            return {}
+            return None
 
         info_tree = html.fromstring(html_list[0])
         catalog_tree = html.fromstring(html_list[1])
 
-        result: dict[str, Any] = {}
-
         # Book metadata
-        book_name = info_tree.xpath(self._BOOK_NAME_XPATH)
-        result["book_name"] = book_name[0].strip() if book_name else ""
+        book_name = self._first_str(info_tree.xpath(self._BOOK_NAME_XPATH))
 
-        book_info3 = info_tree.xpath(self._AUTHOR_INFO_XPATH)
-        result["author"] = book_info3[0].split("/")[0].strip() if book_info3 else ""
-        result["word_count"] = (
-            book_info3[0].split("/")[1].strip()
-            if book_info3 and len(book_info3[0].split("/")) > 1
-            else ""
-        )
+        book_info3_str = self._first_str(info_tree.xpath(self._AUTHOR_INFO_XPATH))
+        author, _, word_count = (p.strip() for p in book_info3_str.partition("/"))
 
-        book_info3_br = info_tree.xpath(self._UPDATE_TIME_XPATH)
-        result["update_time"] = book_info3_br[0].strip() if book_info3_br else ""
+        update_time = self._first_str(info_tree.xpath(self._UPDATE_TIME_XPATH))
 
-        cover_url = info_tree.xpath(self._COVER_URL_XPATH)
-        result["cover_url"] = "https:" + cover_url[0] if cover_url else ""
+        cover_url = "https:" + self._first_str(info_tree.xpath(self._COVER_URL_XPATH))
 
-        serial_status = info_tree.xpath(self._STATUS_XPATH)
-        result["serial_status"] = next(
-            (s for s in serial_status if "完结" in s or "连载" in s), ""
-        )
+        serial_status = self._first_str(info_tree.xpath(self._STATUS_XPATH))
 
-        summary = info_tree.xpath(self._SUMMARY_XPATH)
-        result["summary"] = "".join(summary).strip()
+        summary_elem = info_tree.xpath(self._SUMMARY_XPATH)
+        summary = "".join(summary_elem).strip()
 
         # Chapter structure
         volume_titles = catalog_tree.xpath(self._VOLUME_TITLE_XPATH)
         volume_blocks = catalog_tree.xpath(self._VOLUME_CONTENT_XPATH)
 
-        volumes = []
+        volumes: list[VolumeInfoDict] = []
         for vol_title, vol_block in zip(volume_titles, volume_blocks, strict=False):
-            chapters = []
+            chapters: list[ChapterInfoDict] = []
             for a in vol_block.xpath(self._CHAPTER_LIST_XPATH):
                 href = a.xpath("./@href")[0] if a.xpath("./@href") else ""
                 title = "".join(a.xpath(".//li//text()")).strip()
@@ -107,9 +103,18 @@ class SfacgParser(BaseParser):
                     "chapters": chapters,
                 }
             )
-        result["volumes"] = volumes
 
-        return result
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "word_count": word_count,
+            "serial_status": serial_status,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
 
     def parse_chapter(
         self,
@@ -117,13 +122,6 @@ class SfacgParser(BaseParser):
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
-        """
-        Parse a single chapter page and extract clean text or simplified HTML.
-
-        :param html_list: Raw HTML of the chapter page.
-        :param chapter_id: Identifier of the chapter being parsed.
-        :return: Cleaned chapter content as plain text or minimal HTML.
-        """
         if not html_list:
             return None
         keywords = [
@@ -151,7 +149,7 @@ class SfacgParser(BaseParser):
             raw_text_parts = tree.xpath(self._CHAPTER_TEXT_XPATH)
             content_lines = [txt.strip() for txt in raw_text_parts if txt.strip()]
 
-        content = "\n\n".join(content_lines).strip()
+        content = "\n".join(content_lines).strip()
         if not content:
             return None
 

novel_downloader/core/parsers/shencou.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.shencou
+-------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import etree, html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["shencou"],
+)
+class ShencouParser(BaseParser):
+    """
+    Parser for 神凑轻小说 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if len(html_list) < 2:
+            return None
+
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
+
+        # --- Metadata ---
+        raw_name = self._first_str(info_tree.xpath("//span//a/text()"))
+        book_name = raw_name[:-2] if raw_name.endswith("小说") else raw_name
+
+        author = self._first_str(
+            info_tree.xpath('//td[contains(text(),"小说作者")]/text()'),
+            replaces=[("小说作者：", "")],
+        )
+
+        cover_url = self._first_str(
+            info_tree.xpath('//a[contains(@href,"/files/article/image")]/img/@src')
+        )
+
+        # word count
+        word_count = self._first_str(
+            info_tree.xpath('//td[contains(text(),"全文长度")]/text()'),
+            replaces=[("全文长度：", "")],
+        )
+
+        # update time
+        update_time = self._first_str(
+            info_tree.xpath('//td[contains(text(),"最后更新")]/text()'),
+            replaces=[("最后更新：", "")],
+        )
+
+        # serial status
+        serial_status = self._first_str(
+            info_tree.xpath('//td[contains(text(),"写作进度")]/text()'),
+            replaces=[("写作进度：", "")],
+        )
+
+        # summary
+        raw_detail = self._norm_space(
+            info_tree.xpath('string(//td[@width="80%" and @valign="top"])')
+        )
+        summary = ""
+        if "内容简介：" in raw_detail and "本书公告：" in raw_detail:
+            intro = raw_detail.split("内容简介：", 1)[1]
+            summary = intro.split("本书公告：", 1)[0].strip()
+
+        # --- Catalog / Chapters ---
+        volumes: list[VolumeInfoDict] = []
+        curr_vol: VolumeInfoDict = {"volume_name": "未命名卷", "chapters": []}
+
+        # Walk through volume headers (.zjbox) and lists (.zjlist4) in document order
+        for elem in catalog_tree.xpath(
+            '//div[@class="zjbox"] | //div[@class="zjlist4"]'
+        ):
+            cls_attr = elem.get("class", "")
+            if "zjbox" in cls_attr:
+                # before starting new volume, save the previous if it has chapters
+                if curr_vol["chapters"]:
+                    volumes.append(curr_vol)
+                # start a new volume
+                vol_name = elem.xpath(".//h2/text()")[0].strip()
+                curr_vol = {"volume_name": vol_name, "chapters": []}
+            elif "zjlist4" in cls_attr:
+                # collect all <li><a> entries under this list
+                for a in elem.xpath(".//ol/li/a"):
+                    url = a.get("href").strip()
+                    title = a.text_content().strip()
+                    # '203740.html' -> '203740'
+                    chap_id = url.split(".")[0]
+                    curr_vol["chapters"].append(
+                        {
+                            "title": title,
+                            "url": url,
+                            "chapterId": chap_id,
+                        }
+                    )
+
+        # append last volume if not empty
+        if curr_vol["chapters"]:
+            volumes.append(curr_vol)
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "summary": summary,
+            "volumes": volumes,
+            "word_count": word_count,
+            "serial_status": serial_status,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+        title = self._first_str(tree.xpath("//h1/text()"))
+        if not title:
+            return None
+
+        # strip book-name prefix if present
+        bc = tree.xpath('//div[@id="breadCrumb"]//a/text()')
+        if len(bc) >= 2:
+            book_name = bc[1].strip()
+            title = title.removeprefix(book_name).lstrip(" ：:–—-").strip()
+
+        anchors = tree.xpath('//div[@id="BookSee_Right"]')
+        if not anchors:
+            return None
+        marker = anchors[0]
+
+        lines: list[str] = []
+
+        def _append_text(text: str) -> None:
+            for ln in text.replace("\xa0", " ").splitlines():
+                ln2 = ln.strip()
+                if ln2:
+                    lines.append(ln2)
+
+        if marker.tail:
+            _append_text(marker.tail)
+
+        # 4. Walk through siblings until <!--over-->
+        node = marker
+        while True:
+            sib = node.getnext()
+            if sib is None:
+                break
+            node = sib
+
+            # Stop on the closing comment
+            if isinstance(sib, etree._Comment) and "over" in (sib.text or ""):
+                break
+
+            # Process comment tails (e.g. <!--go--> tail)
+            if isinstance(sib, etree._Comment):
+                if sib.tail:
+                    _append_text(sib.tail)
+                continue
+
+            if isinstance(sib, html.HtmlElement):
+                # tag = sib.tag.lower()
+                tag = str(sib.tag).lower()
+                cls = sib.get("class", "") or ""
+
+                if tag == "div" and "divimage" in cls:
+                    srcs = sib.xpath(".//img/@src")
+                    if srcs:
+                        lines.append(f'<img src="{srcs[0]}" />')
+                    # text after the div
+                    if sib.tail:
+                        _append_text(sib.tail)
+                    continue
+
+                if tag == "br":
+                    if sib.tail:
+                        _append_text(sib.tail)
+                    continue
+
+                text = sib.text_content()
+                _append_text(text)
+                if sib.tail:
+                    _append_text(sib.tail)
+                continue
+
+        content = "\n".join(lines)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "shencou"},
+        }

novel_downloader/core/parsers/shuhaige.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.shuhaige
+--------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["shuhaige"],
+)
+class ShuhaigeParser(BaseParser):
+    """
+    Parser for 书海阁小说网 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath('//div[@id="info"]/h1/text()'))
+        author = self._first_str(tree.xpath('//div[@id="info"]/p[1]/a/text()'))
+
+        cover_url = self._first_str(tree.xpath('//div[@id="fmimg"]/img/@src'))
+
+        update_time = self._first_str(
+            tree.xpath('//div[@id="info"]/p[3]/text()'),
+            replaces=[("最后更新：", "")],
+        )
+
+        summary = self._first_str(tree.xpath('//div[@id="intro"]/p[1]/text()'))
+
+        book_type = self._first_str(tree.xpath('//div[@class="con_top"]/a[2]/text()'))
+        tags = [book_type] if book_type else []
+
+        chapters: list[ChapterInfoDict] = [
+            {
+                "title": (a.text or "").strip(),
+                "url": (a.get("href") or "").strip(),
+                "chapterId": (a.get("href") or "").rsplit("/", 1)[-1].split(".", 1)[0],
+            }
+            for a in tree.xpath(
+                '//div[@id="list"]/dl/dt[contains(., "正文")]/following-sibling::dd/a'
+            )
+        ]
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath('//div[@class="bookname"]/h1/text()'))
+        if not title:
+            title = f"第 {chapter_id} 章"
+
+        content_elem = tree.xpath('//div[@id="content"]')
+        if not content_elem:
+            return None
+        paragraphs = [
+            "".join(p.itertext()).strip() for p in content_elem[0].xpath(".//p")
+        ]
+        if paragraphs and "www.shuhaige.net" in paragraphs[-1]:
+            paragraphs.pop()
+
+        content = "\n".join(paragraphs)
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "shuhaige"},
+        }

novel_downloader/core/parsers/tongrenquan.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.tongrenquan
+-----------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["tongrenquan"],
+)
+class TongrenquanParser(BaseParser):
+    """
+    Parser for 同人圈 book pages.
+    """
+
+    BASE_URL = "https://www.tongrenquan.org"
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # Metadata
+        book_name = self._first_str(tree.xpath('//div[@class="infos"]/h1/text()'))
+        author = self._first_str(
+            tree.xpath('//div[@class="date"]/span/text()'),
+            replaces=[("作者：", "")],
+        )
+        cover_url = self.BASE_URL + self._first_str(
+            tree.xpath('//div[@class="pic"]//img/@src')
+        )
+        update_time = self._first_str(
+            tree.xpath('//div[@class="date"]/text()'),
+            replaces=[("日期：", "")],
+        )
+
+        # Summary (collapse text within the <p> tag)
+        paras = tree.xpath('//div[@class="infos"]/p//text()')
+        summary = "\n".join(p.strip() for p in paras if p.strip())
+
+        # Chapters extraction
+        chapters: list[ChapterInfoDict] = []
+        for a in tree.xpath('//div[contains(@class,"book_list")]//ul//li/a'):
+            url = a.get("href", "").strip()
+            title = a.text_content().strip()
+            # General pattern: /category/bookId/chapterId.html
+            # '/tongren/7562/462.html' -> '462'
+            chapter_id = url.rstrip(".html").split("/")[-1]
+            chapters.append({"title": title, "url": url, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": ["同人小说"],
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        raw_title = self._first_str(
+            tree.xpath('//div[contains(@class,"read_chapterName")]//h1/text()')
+        )
+
+        book_name = self._first_str(
+            tree.xpath('//div[contains(@class,"readTop")]//a[last()]/text()')
+        )
+
+        title = raw_title.replace(book_name, "").strip()
+
+        # Extract paragraphs of content
+        paras = tree.xpath('//div[contains(@class,"read_chapterDetail")]/p')
+        texts = [p.text_content().strip() for p in paras if p.text_content().strip()]
+        content = "\n".join(texts)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "tongrenquan"},
+        }