novel-downloader 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/core/fetchers/common/browser.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.fetchers.common.browser
----------------------------------------------
-
-"""
-
-from typing import Any
-
-from novel_downloader.core.fetchers.base import BaseBrowser
-from novel_downloader.models import FetcherConfig, SiteProfile
-
-
-class CommonBrowser(BaseBrowser):
-    """
-    A common async browser for handling site-specific HTTP requests.
-    """
-
-    def __init__(
-        self,
-        site: str,
-        profile: SiteProfile,
-        config: FetcherConfig,
-        reuse_page: bool = False,
-        **kwargs: Any,
-    ) -> None:
-        super().__init__(site, config, reuse_page, **kwargs)
-        self._profile = profile
-
-    async def get_book_info(
-        self,
-        book_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML of the book info page asynchronously.
-
-        :param book_id: The book identifier.
-        :return: The page content as a string.
-        """
-        url = self.book_info_url(book_id=book_id)
-        return [await self.fetch(url, **kwargs)]
-
-    async def get_book_chapter(
-        self,
-        book_id: str,
-        chapter_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML of a single chapter asynchronously.
-
-        :param book_id: The book identifier.
-        :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
-        """
-        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
-        return [await self.fetch(url, **kwargs)]
-
-    def book_info_url(self, book_id: str) -> str:
-        """
-        Construct the URL for fetching a book's info page.
-
-        :param book_id: The identifier of the book.
-        :return: Fully qualified URL for the book info page.
-        """
-        return self._profile["book_info_url"].format(book_id=book_id)
-
-    def chapter_url(self, book_id: str, chapter_id: str) -> str:
-        """
-        Construct the URL for fetching a specific chapter.
-
-        :param book_id: The identifier of the book.
-        :param chapter_id: The identifier of the chapter.
-        :return: Fully qualified chapter URL.
-        """
-        return self._profile["chapter_url"].format(
-            book_id=book_id, chapter_id=chapter_id
-        )

novel_downloader/core/fetchers/common/session.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.fetchers.common.session
----------------------------------------------
-
-"""
-
-from typing import Any
-
-from novel_downloader.core.fetchers.base import BaseSession
-from novel_downloader.models import FetcherConfig, SiteProfile
-
-
-class CommonSession(BaseSession):
-    """
-    A common async session for handling site-specific HTTP requests.
-    """
-
-    def __init__(
-        self,
-        site: str,
-        profile: SiteProfile,
-        config: FetcherConfig,
-        cookies: dict[str, str] | None = None,
-        **kwargs: Any,
-    ) -> None:
-        super().__init__(site, config, cookies, **kwargs)
-        self._profile = profile
-
-    async def get_book_info(
-        self,
-        book_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML of the book info page asynchronously.
-
-        :param book_id: The book identifier.
-        :return: The page content as a string.
-        """
-        url = self.book_info_url(book_id=book_id)
-        return [await self.fetch(url, **kwargs)]
-
-    async def get_book_chapter(
-        self,
-        book_id: str,
-        chapter_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML of a single chapter asynchronously.
-
-        :param book_id: The book identifier.
-        :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
-        """
-        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
-        return [await self.fetch(url, **kwargs)]
-
-    def book_info_url(self, book_id: str) -> str:
-        """
-        Construct the URL for fetching a book's info page.
-
-        :param book_id: The identifier of the book.
-        :return: Fully qualified URL for the book info page.
-        """
-        return self._profile["book_info_url"].format(book_id=book_id)
-
-    def chapter_url(self, book_id: str, chapter_id: str) -> str:
-        """
-        Construct the URL for fetching a specific chapter.
-
-        :param book_id: The identifier of the book.
-        :param chapter_id: The identifier of the chapter.
-        :return: Fully qualified chapter URL.
-        """
-        return self._profile["chapter_url"].format(
-            book_id=book_id, chapter_id=chapter_id
-        )

novel_downloader/core/parsers/biquge/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.biquge
-------------------------------------
-
-"""
-
-from .main_parser import BiqugeParser
-
-__all__ = ["BiqugeParser"]

novel_downloader/core/parsers/common/__init__.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.common
-------------------------------------
-
-This module provides a CommonParser class that implements
-general-purpose parsing logic for extracting novel metadata
-and chapter content based on site-specific rules.
-"""
-
-from .main_parser import CommonParser
-
-__all__ = ["CommonParser"]

novel_downloader/core/parsers/common/helper.py

@@ -1,323 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.common.helper
--------------------------------------------
-
-Shared utility functions for parsing Common pages.
-"""
-
-import logging
-import re
-from collections.abc import Iterable, Iterator
-from typing import Any, cast
-
-from bs4 import BeautifulSoup, Tag
-
-from novel_downloader.models import (
-    BookInfoRules,
-    FieldRules,
-    RuleStep,
-    VolumesRules,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def html_to_soup(html_str: str) -> BeautifulSoup:
-    """
-    Convert an HTML string to a BeautifulSoup object with fallback.
-
-    :param html_str: Raw HTML string.
-    :return: Parsed BeautifulSoup object.
-    """
-    try:
-        return BeautifulSoup(html_str, "lxml")
-    except Exception as e:
-        logger.warning("[Parser] lxml parse failed, falling back: %s", e)
-        return BeautifulSoup(html_str, "html.parser")
-
-
-class HTMLExtractor:
-    """
-    HTML extraction engine that applies a sequence of RuleSteps to
-    pull data out of a page.
-    """
-
-    def __init__(self, html: str):
-        self._html = html
-        self._soup = html_to_soup(html)
-
-    def extract_book_info(self, rules: BookInfoRules) -> dict[str, Any]:
-        """
-        Extract structured book information from HTML according to the given rules.
-
-        Only non-empty fields in the rules are processed.
-
-        :param rules: Extraction configuration specifying how to extract.
-        :return: A dictionary containing extracted book information.
-        """
-        book_info: dict[str, Any] = {}
-
-        for field_name, field_rules in rules.items():
-            if field_rules is None:
-                continue
-
-            if field_name == "volumes":
-                book_info[field_name] = self.extract_volumes_structure(
-                    cast(VolumesRules, field_rules)
-                )
-            else:
-                steps = cast(FieldRules, field_rules)["steps"]
-                book_info[field_name] = self.extract_field(steps)
-
-        return book_info
-
-    def extract_field(self, steps: list[RuleStep]) -> str:
-        """
-        Execute a list of extraction steps on the given HTML.
-
-        - If any step yields None, stops processing further steps.
-        - At the end, always returns a str:
-        * If current is a list, converts items to text and joins with '\n'.
-        * If current is a Tag, extracts its .get_text().
-        * Else, uses str().
-        """
-
-        def flatten_list(items: Iterable[Any]) -> Iterator[Any]:
-            for item in items:
-                if isinstance(item, list):
-                    yield from flatten_list(item)
-                else:
-                    yield item
-
-        def to_text(item: Any) -> str:
-            if isinstance(item, Tag):
-                return str(item.get_text().strip())
-            return str(item).strip()
-
-        current: Any = self._soup
-
-        for step in steps:
-            t = step.get("type")
-            if t == "select_one":
-                sel = step.get("selector")
-                current = current.select_one(sel) if sel else None
-
-            elif t == "select":
-                sel = step.get("selector")
-                lst = current.select(sel) if sel else []
-                idx = step.get("index")
-                current = lst[idx] if idx is not None and idx < len(lst) else lst
-
-            elif t == "exclude":
-                sel = step.get("selector")
-                for elem in current.select(sel or ""):
-                    elem.decompose()
-
-            elif t == "find":
-                nm = step.get("name")
-                attrs = step.get("attrs") or {}
-                current = current.find(nm, attrs=attrs)
-
-            elif t == "find_all":
-                nm = step.get("name")
-                attrs = step.get("attrs") or {}
-                lst = current.find_all(nm, attrs=attrs, limit=step.get("limit"))
-                idx = step.get("index")
-                current = lst[idx] if idx is not None and idx < len(lst) else lst
-
-            elif t == "text":
-                if isinstance(current, list):
-                    current = [elem.get_text() for elem in current]
-                elif isinstance(current, Tag):
-                    current = current.get_text()
-
-            elif t == "strip":
-                chars = step.get("chars")
-                if isinstance(current, list):
-                    current = [c.strip(chars) for c in current]
-                elif isinstance(current, str):
-                    current = current.strip(chars)
-
-            elif t == "regex":
-                txt = str(current or "")
-                pat = step.get("pattern") or ""
-                flags = step.get("flags")
-                flags = flags if flags is not None else 0
-                match = re.compile(pat, flags).search(txt)
-                if match:
-                    template = step.get("template")
-                    if template:
-                        s = template
-                        for i in range(1, len(match.groups()) + 1):
-                            s = s.replace(f"${i}", match.group(i) or "")
-                        current = s
-                    else:
-                        grp = step.get("group")
-                        grp = grp if grp is not None else 0
-                        current = match.group(grp)
-                else:
-                    current = ""
-
-            elif t == "replace":
-                old = step.get("old")
-                old = old if old is not None else ""
-
-                new = step.get("new")
-                new = new if new is not None else ""
-
-                cnt = step.get("count")
-                cnt = cnt if cnt is not None else -1
-
-                if isinstance(current, list):
-                    current = [c.replace(old, new, cnt) for c in current]
-                elif isinstance(current, str):
-                    current = current.replace(old, new, cnt)
-
-            elif t == "split":
-                sep = step.get("sep", "")
-                idx = step.get("index")
-                idx = idx if idx is not None else 0
-                parts = (current or "").split(sep)
-                current = parts[idx] if idx < len(parts) else ""
-
-            elif t == "join":
-                sep = step.get("sep")
-                sep = sep if sep is not None else ""
-                if isinstance(current, list):
-                    current = sep.join(current)
-
-            elif t == "attr":
-                name = step.get("attr") or ""
-                if isinstance(current, list):
-                    current = [elem.get(name, "") for elem in current]
-                elif isinstance(current, Tag):
-                    current = current.get(name, "")
-
-            else:
-                raise ValueError(f"Unsupported step type: {t}")
-
-            if current is None:
-                break
-
-        # Final normalization
-        if isinstance(current, list):
-            flat = list(flatten_list(current))
-            texts = [to_text(x) for x in flat if x is not None]
-            return "\n".join(texts)
-        if isinstance(current, Tag):
-            return str(current.get_text().strip())
-        return str(current or "").strip()
-
-    def extract_mixed_volumes(self, volume_rule: VolumesRules) -> list[dict[str, Any]]:
-        """
-        Special mode: mixed <volume> and <chapter> under same parent.
-        (e.g., dt / dd pattern in BiQuGe)
-        """
-        list_selector = volume_rule.get("list_selector")
-        volume_selector = volume_rule.get("volume_selector")
-        volume_name_steps = volume_rule.get("volume_name_steps")
-        chapter_selector = volume_rule["chapter_selector"]
-        chapter_steps_list = volume_rule["chapter_steps"]
-
-        if not (
-            list_selector and volume_selector and chapter_selector and volume_name_steps
-        ):
-            raise ValueError(
-                "volume_mode='mixed' 时, 必须提供 list_selector, volume_selector, "
-                "chapter_selector 和 volume_name_steps"
-            )
-
-        volumes: list[dict[str, Any]] = []
-        current_volume: dict[str, Any] | None = None
-        if not chapter_steps_list:
-            chapter_steps_list = []
-        chapter_info_steps = {item["key"]: item["steps"] for item in chapter_steps_list}
-
-        list_area = self._soup.select_one(list_selector)
-        if not list_area:
-            raise ValueError(f"找不到 list_selector: {list_selector}")
-
-        for elem in list_area.find_all(
-            [volume_selector, chapter_selector], recursive=True
-        ):
-            if not isinstance(elem, Tag):
-                continue
-            if elem.name == volume_selector:
-                extractor = HTMLExtractor(str(elem))
-                volume_name = extractor.extract_field(volume_name_steps)
-                current_volume = {"volume_name": volume_name, "chapters": []}
-                volumes.append(current_volume)
-
-            elif elem.name == chapter_selector and current_volume is not None:
-                chap_extractor = HTMLExtractor(str(elem))
-                chapter_data = {}
-                for field, steps in chapter_info_steps.items():
-                    chapter_data[field] = chap_extractor.extract_field(steps)
-                current_volume["chapters"].append(chapter_data)
-
-        return volumes
-
-    def extract_volume_blocks(self, volume_rule: VolumesRules) -> list[dict[str, Any]]:
-        volume_selector = volume_rule.get("volume_selector")
-        volume_name_steps = volume_rule.get("volume_name_steps")
-        chapter_selector = volume_rule["chapter_selector"]
-        chapter_steps_list = volume_rule["chapter_steps"]
-        if not (volume_selector and volume_name_steps):
-            raise ValueError(
-                "has_volume=True 时, 必须提供 volume_selector 和 volume_name_steps"
-            )
-        volumes = []
-        chapter_info_steps = {item["key"]: item["steps"] for item in chapter_steps_list}
-        for vol in self._soup.select(volume_selector):
-            extractor = HTMLExtractor(str(vol))
-            volume_name = extractor.extract_field(volume_name_steps)
-
-            chapters = []
-            for chap in vol.select(chapter_selector):
-                chap_extractor = HTMLExtractor(str(chap))
-                chapter_data = {}
-                for field, steps in chapter_info_steps.items():
-                    chapter_data[field] = chap_extractor.extract_field(steps)
-                chapters.append(chapter_data)
-
-            volumes.append({"volume_name": volume_name, "chapters": chapters})
-
-        return volumes
-
-    def extract_flat_chapters(self, volume_rule: VolumesRules) -> list[dict[str, Any]]:
-        chapter_selector = volume_rule["chapter_selector"]
-        chapter_steps_list = volume_rule["chapter_steps"]
-        volume_selector = volume_rule.get("volume_selector")
-        volumes = []
-        chapter_info_steps = {item["key"]: item["steps"] for item in chapter_steps_list}
-
-        if volume_selector:
-            candidates = self._soup.select(volume_selector)
-        else:
-            candidates = [self._soup]
-
-        all_chapters = []
-        for area in candidates:
-            for chap in area.select(chapter_selector):
-                chap_extractor = HTMLExtractor(str(chap))
-                chapter_data = {}
-                for field, steps in chapter_info_steps.items():
-                    chapter_data[field] = chap_extractor.extract_field(steps)
-                all_chapters.append(chapter_data)
-
-        volumes.append({"volume_name": "未分卷", "chapters": all_chapters})
-
-        return volumes
-
-    def extract_volumes_structure(
-        self, volume_rule: VolumesRules
-    ) -> list[dict[str, Any]]:
-        volume_mode = volume_rule.get("volume_mode", "normal")
-        if volume_mode == "mixed":
-            return self.extract_mixed_volumes(volume_rule)
-
-        if volume_rule.get("has_volume", True):
-            return self.extract_volume_blocks(volume_rule)
-        else:
-            return self.extract_flat_chapters(volume_rule)

novel_downloader/core/parsers/common/main_parser.py

@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.common.main_parser
-------------------------------------------------
-
-This package provides parsing components for handling
-Common pages.
-"""
-
-from typing import Any
-
-from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.models import (
-    ChapterDict,
-    ParserConfig,
-    SiteRules,
-)
-
-# from .helper import HTMLExtractor
-
-
-class CommonParser(BaseParser):
-    """
-    CommonParser extends BaseParser to support site-specific parsing rules.
-
-    It accepts additional site information and site-specific rules during initialization
-    """
-
-    def __init__(self, config: ParserConfig, site: str, site_rule: SiteRules):
-        """
-        Initialize the parser with configuration, site name, and site-specific rules.
-
-        :param config: ParserConfig object controlling parsing behavior.
-        :param site: Name of the site this parser is targeting.
-        :param site_rule: SiteRules object containing parsing rules for the site.
-        """
-        super().__init__(config)
-        self._site = site
-        self._site_rule = site_rule
-
-    def parse_book_info(
-        self,
-        html_list: list[str],
-        **kwargs: Any,
-    ) -> dict[str, Any]:
-        """
-        Parse a book info page and extract metadata and chapter structure.
-
-        :param html_list: Raw HTML of the book info page.
-        :return: Parsed metadata and chapter structure as a dictionary.
-        """
-        if not html_list:
-            return {}
-        # extractor = HTMLExtractor(html_list[0])
-        # rules = self._site_rule["book_info"]
-        # return extractor.extract_book_info(rules)
-        return {}
-
-    def parse_chapter(
-        self,
-        html_list: list[str],
-        chapter_id: str,
-        **kwargs: Any,
-    ) -> ChapterDict | None:
-        """
-        Parse a single chapter page and extract clean text or simplified HTML.
-
-        :param html_list: Raw HTML of the chapter page.
-        :param chapter_id: Identifier of the chapter being parsed.
-        :return: Cleaned chapter content as plain text or minimal HTML.
-        """
-        if not html_list:
-            return None
-        # extractor = HTMLExtractor(html_list[0])
-        # chapter_rules = self._site_rule["chapter"]
-
-        # # 必须有正文内容
-        # content_steps = chapter_rules.get("content")
-        # if not content_steps:
-        #     raise ValueError(f"No chapter content steps for site: {self._site}")
-
-        # title_steps = chapter_rules.get("title")
-        # title = extractor.extract_field(title_steps["steps"]) if title_steps else ""
-        # content = extractor.extract_field(content_steps["steps"])
-        # if not content:
-        #     return None
-
-        # return {
-        #     "id": chapter_id,
-        #     "title": title or "Untitled",
-        #     "content": content,
-        #     "extra": {
-        #         "site": self._site,
-        #     },
-        # }
-        return None
-
-    @property
-    def site(self) -> str:
-        """Return the site name."""
-        return self._site
-
-    @property
-    def site_rule(self) -> SiteRules:
-        """Return the site-specific rules."""
-        return self._site_rule

novel_downloader/core/parsers/esjzone/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.esjzone
--------------------------------------
-
-"""
-
-from .main_parser import EsjzoneParser
-
-__all__ = ["EsjzoneParser"]

novel_downloader/core/parsers/linovelib/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.linovelib
----------------------------------------
-
-"""
-
-from .main_parser import LinovelibParser
-
-__all__ = ["LinovelibParser"]

novel_downloader/core/parsers/qianbi/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.qianbi
-------------------------------------
-
-"""
-
-from .main_parser import QianbiParser
-
-__all__ = ["QianbiParser"]

novel_downloader/core/parsers/sfacg/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.sfacg
------------------------------------
-
-"""
-
-from .main_parser import SfacgParser
-
-__all__ = ["SfacgParser"]

novel_downloader/core/parsers/yamibo/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.yamibo
-------------------------------------
-
-"""
-
-from .main_parser import YamiboParser
-
-__all__ = ["YamiboParser"]

novel_downloader/models/browser.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.models.browser
--------------------------------
-
-"""
-
-from pathlib import Path
-from typing import TypedDict
-
-from playwright.async_api import ViewportSize
-
-
-class NewContextOptions(TypedDict, total=False):
-    user_agent: str
-    locale: str
-    storage_state: Path
-    viewport: ViewportSize
-    java_script_enabled: bool
-    ignore_https_errors: bool
-    extra_http_headers: dict[str, str]