parsehub 2.0.13__tar.gz → 2.0.14__tar.gz

{parsehub-2.0.13/src/parsehub.egg-info → parsehub-2.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.13
+Version: 2.0.14
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT

{parsehub-2.0.13 → parsehub-2.0.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.0.13"
+version = "2.0.14"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"

{parsehub-2.0.13 → parsehub-2.0.14}/src/parsehub/parsers/parser/douyin.py

@@ -21,7 +21,7 @@ from ..base.base import BaseParser
 class DouyinParser(BaseParser):
     __platform__ = Platform.DOUYIN
     __supported_type__ = ["视频", "图文"]
-    __match__ = r"^(http(s)?://)?.+douyin.com/(?!share/user).+|^(http(s)?://)?.+tiktok.com/.+"
+    __match__ = r"^(http(s)?://)?.+douyin.com/(?!share/user|qishui).+|^(http(s)?://)?.+tiktok.com/.+"
     __redirect_keywords__ = ["v.douyin", "vt.tiktok", "iesdouyin"]
     __reserved_parameters__ = ["modal_id"]
 

parsehub-2.0.14/src/parsehub/parsers/parser/tieba.py

@@ -0,0 +1,78 @@
+from typing import Union
+
+import httpx
+
+from ...provider_api.tieba import TieBa, TieBaError, TieBaPostType
+from ...types import AniRef, ImageParseResult, ImageRef, ParseError, Platform, VideoParseResult, VideoRef
+from ..base.base import BaseParser
+
+
+class TieBaParser(BaseParser):
+    __platform__ = Platform.TIEBA
+    __supported_type__ = ["视频", "图文"]
+    __match__ = r"^(http(s)?://)?.+tieba.baidu.com/p/\d+"
+
+    async def _do_parse(self, raw_url: str) -> Union["ImageParseResult", "VideoParseResult"]:
+        try:
+            tb = await TieBa(self.proxy).parse(raw_url)
+        except TieBaError as e:
+            raise ParseError(e.msg if e.msg else "贴吧解析失败: 未知错误") from e
+        except Exception as e:
+            raise ParseError("贴吧解析失败: 未知错误") from e
+
+        match tb.type:
+            case TieBaPostType.VIDEO:
+                return VideoParseResult(
+                    title=tb.title,
+                    video=VideoRef(
+                        url=tb.media.url,
+                        thumb_url=tb.media.thumb_url,
+                        width=tb.media.width,
+                        height=tb.media.height,
+                        duration=tb.media.duration,
+                    ),
+                    content=tb.content,
+                )
+
+            case TieBaPostType.PHOTO:
+                images = []
+                if tb.media:
+                    for i in tb.media:
+                        async with httpx.AsyncClient(proxy=self.proxy) as cli:
+                            try:
+                                r = await cli.head(i.url)
+                                r.raise_for_status()
+                            except Exception:
+                                images.append(
+                                    ImageRef(
+                                        url=i.url,
+                                        thumb_url=i.thumb_url,
+                                        width=i.width,
+                                        height=i.height,
+                                    )
+                                )
+                            else:
+                                headers = r.headers
+                                if (t := headers.get("content-type")) and "gif" in t:
+                                    images.append(
+                                        AniRef(
+                                            url=i.url,
+                                            thumb_url=i.thumb_url,
+                                            width=i.width,
+                                            height=i.height,
+                                        )
+                                    )
+                                else:
+                                    images.append(
+                                        ImageRef(
+                                            url=i.url,
+                                            thumb_url=i.thumb_url,
+                                            width=i.width,
+                                            height=i.height,
+                                        )
+                                    )
+
+                return ImageParseResult(title=tb.title, content=tb.content, photo=images)
+
+
+__all__ = ["TieBaParser"]

parsehub-2.0.14/src/parsehub/provider_api/tieba.py

@@ -0,0 +1,153 @@
+import hashlib
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Self
+
+import httpx
+
+
+class TieBa:
+    def __init__(self, proxy: str | None = None):
+        self.proxy = proxy
+
+    async def parse(self, url: str) -> "TieBaPost":
+        data = await self.fetch_post_data(url)
+        return TieBaPost.parse(data)
+
+    @staticmethod
+    def gen_sign(params: dict):
+        items = sorted(params.items())
+        base_str = "".join([f"{k}={v}" for k, v in items])
+        salt = "36770b1f34c9bbf2e7d1a99d2b82fa9e"
+        return hashlib.md5((base_str + salt).encode("utf-8")).hexdigest()
+
+    async def fetch_tbs(self) -> str:
+        async with httpx.AsyncClient(proxy=self.proxy) as cli:
+            result = await cli.get("http://tieba.baidu.com/dc/common/tbs")
+            result.raise_for_status()
+        result = result.json()
+        if tbs := result.get("tbs"):
+            return tbs
+        raise TieBaError("获取 tbs 失败")
+
+    @staticmethod
+    def get_kz(url: str) -> str:
+        if match := re.search(r"/p/(\d+)", url):
+            return match.group(1)
+        raise ValueError("无法从 URL 中提取帖子 ID")
+
+    async def fetch_post_data(self, url: str) -> dict:
+        kz = self.get_kz(url)
+        tbs = await self.fetch_tbs()
+        data = {
+            "pn": "1",
+            "lz": "0",
+            "r": "2",
+            "mark_type": "0",
+            "back": "0",
+            "fr": "personalize_page",
+            "kz": kz,
+            "session_request_times": "1",
+            "tbs": tbs,
+            "subapp_type": "pc",
+            "_client_type": "20",
+        }
+        data["sign"] = self.gen_sign(data)
+        async with httpx.AsyncClient(proxy=self.proxy, timeout=30) as cli:
+            result = await cli.post("https://tieba.baidu.com/c/f/pb/page_pc", data=data)
+            result.raise_for_status()
+            result = result.json()
+        if result["error_code"]:
+            raise TieBaError(em if (em := result["error_msg"]) else "获取帖子内容失败")
+        return result
+
+
+class TieBaPostType(Enum):
+    PHOTO = "PHOTO"
+    VIDEO = "VIDEO"
+
+
+@dataclass
+class TieBaVideo:
+    url: str
+    thumb_url: str | None = None
+    width: int = 0
+    height: int = 0
+    duration: int = 0
+
+
+@dataclass
+class TieBaPhoto:
+    url: str
+    thumb_url: str | None = None
+    width: int = 0
+    height: int = 0
+
+
+@dataclass
+class TieBaPost:
+    type: TieBaPostType
+    title: str
+    content: str
+    media: list[TieBaPhoto] | TieBaVideo | None = None
+
+    @classmethod
+    def parse(cls, data: dict) -> Self:
+        thread = data["thread"]
+        origin_thread_info = thread["origin_thread_info"]
+
+        # title
+        title = origin_thread_info["title"]
+
+        # content
+        origin_content = origin_thread_info["content"]
+        content_list: list[str] = []
+        for oc in origin_content:
+            oc_type = oc["type"]
+            match oc_type:
+                case 0:
+                    content_list.append(oc["text"])
+        content = "\n".join(content_list)
+
+        # media
+        media = []
+        if origin_media := origin_thread_info.get("media"):
+            post_type = TieBaPostType.PHOTO
+            for om in origin_media:
+                media.append(
+                    TieBaPhoto(
+                        url=om["big_pic"],
+                        thumb_url=om["small_pic"],
+                        width=om["width"],
+                        height=om["height"],
+                    )
+                )
+
+        elif video_info := thread.get("video_info"):
+            post_type = TieBaPostType.VIDEO
+            media.append(
+                TieBaVideo(
+                    url=video_info["video_url"],
+                    thumb_url=video_info["thumbnail_url"],
+                    width=video_info["video_width"],
+                    height=video_info["video_height"],
+                    duration=video_info["video_duration"],
+                )
+            )
+        else:
+            post_type = TieBaPostType.PHOTO
+
+        m = media[0] if post_type == TieBaPostType.VIDEO else media if media else None
+        return TieBaPost(
+            type=post_type,
+            title=title,
+            content=content,
+            media=m,
+        )
+
+
+class TieBaError(Exception):
+    def __init__(self, msg: str):
+        self.msg = msg
+        super().__init__(msg)

{parsehub-2.0.13 → parsehub-2.0.14/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.13
+Version: 2.0.14
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT

parsehub-2.0.13/src/parsehub/parsers/parser/tieba.py

@@ -1,25 +0,0 @@
-from typing import Union
-
-from ...provider_api.tieba import TieBa
-from ...types import ImageParseResult, ParseError, Platform, VideoParseResult
-from ..base.base import BaseParser
-
-
-class TieBaParser(BaseParser):
-    __platform__ = Platform.TIEBA
-    __supported_type__ = ["视频", "图文"]
-    __match__ = r"^(http(s)?://)?.+tieba.baidu.com/p/\d+"
-
-    async def _do_parse(self, raw_url: str) -> Union["ImageParseResult", "VideoParseResult"]:
-        try:
-            tb = await TieBa(self.proxy).parse(raw_url)
-        except Exception as e:
-            raise ParseError("贴吧解析失败") from e
-
-        if tb.video_url:
-            return VideoParseResult(title=tb.title, video=tb.video_url, content=tb.content)
-        else:
-            return ImageParseResult(title=tb.title, photo=tb.img_url, content=tb.content)
-
-
-__all__ = ["TieBaParser"]

parsehub-2.0.13/src/parsehub/provider_api/tieba.py

@@ -1,74 +0,0 @@
-from dataclasses import dataclass
-
-import httpx
-from bs4 import BeautifulSoup
-from httpx import Response
-
-
-class TieBa:
-    def __init__(self, proxy: str | None = None):
-        self.proxy = proxy
-
-    @staticmethod
-    def _parse_out_the_body(text):
-        soup = BeautifulSoup(str(text), "lxml")
-        div_tag = soup.find_all("div")
-        [img.extract() for img in soup.find_all("img")]
-        [i.unwrap() for i in div_tag]
-        text = soup.text.strip()
-        # text = re.sub(
-        #     r"(<br/><br/>)+|点击展开，查看完整图片|<i.*></i>", "", str(soup)
-        # ).strip()
-        # text = re.sub(r'<span class="apc_src_wrapper">视频来自：.*</span>', "", text)
-        return text
-
-    @staticmethod
-    async def get_tieba_img_url(html: Response):
-        """获取帖子中所有图片的URL"""
-        soup = BeautifulSoup(html.text, "lxml")
-        d_post_content_firstfloor = soup.find("div", {"class": "d_post_content_firstfloor"})
-        img_tags = d_post_content_firstfloor.find_all("img", {"class": "BDE_Image"})
-        return [img["src"] for img in img_tags if "src" in img.attrs]
-
-    @staticmethod
-    async def get_tieba_video_url(html: Response):
-        """获取帖子中所有视频的URL"""
-        soup = BeautifulSoup(html.text, "lxml")
-        d_post_content_firstfloor = soup.find("div", {"class": "d_post_content_firstfloor"})
-
-        if video_tags := d_post_content_firstfloor.find("embed", {"class": "BDE_Flash"}):
-            return video_tags["data-video"]
-        return None
-
-    async def get_the_content(self, html: Response):
-        """获取帖子的标题和内容"""
-        soup = BeautifulSoup(html.text, "lxml")
-        title = soup.find("h3", {"class": ["core_title_txt", "pull-left", "text-overflow"]}) or soup.find(
-            "h1", {"class": "core_title_txt"}
-        )
-        if not title:
-            raise Exception("未获取到标题内容")
-        title = title.text.strip()
-        content = soup.find("div", {"class": ["d_post_content", "j_d_post_content"]})
-        content = self._parse_out_the_body(content)
-        return title, content
-
-    async def get_html(self, t_url) -> Response:
-        async with httpx.AsyncClient(proxy=self.proxy) as c:
-            return await c.get(t_url, headers={"User-Agent": "Mozilla5.0/"}, timeout=15)
-
-    async def parse(self, t_url) -> "TieBaPost":
-        res = await self.get_html(t_url)
-
-        title, content = await self.get_the_content(res)
-        img_url = await self.get_tieba_img_url(res)
-        video_url = await self.get_tieba_video_url(res)
-        return TieBaPost(title, content, img_url, video_url)
-
-
-@dataclass
-class TieBaPost:
-    title: str
-    content: str
-    img_url: list
-    video_url: str = None