parsehub 2.0.3__tar.gz → 2.0.5__tar.gz

{parsehub-2.0.3/src/parsehub.egg-info → parsehub-2.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.3
+Version: 2.0.5
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -119,6 +119,25 @@ print(result)
 
 ## 🔑 高级用法
 
+### 下载进度回调
+
+```python
+from parsehub import ParseHub
+
+
+class ProgressTracker:
+    async def __call__(self, current: int, total: int, unit: str, *args, task_name: str = "", **kwargs):
+        print(f"[{task_name}] {current}/{total} ({unit})")
+
+
+result = ParseHub().download_sync(
+    "https://example.com",
+    callback=ProgressTracker(),
+    callback_args=("extra_arg",),
+    callback_kwargs={"task_name": "demo"},
+)
+```
+
 ### Cookie 登录 & 代理
 
 部分平台的内容需要登录才能访问，通过 Cookie 即可解锁：
@@ -126,7 +145,7 @@ print(result)
 ```python
 from parsehub import ParseHub
 
-ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890",)
+ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890", )
 ```
 
 Cookie 支持多种格式传入：

{parsehub-2.0.3 → parsehub-2.0.5}/README.md

@@ -82,6 +82,25 @@ print(result)
 
 ## 🔑 高级用法
 
+### 下载进度回调
+
+```python
+from parsehub import ParseHub
+
+
+class ProgressTracker:
+    async def __call__(self, current: int, total: int, unit: str, *args, task_name: str = "", **kwargs):
+        print(f"[{task_name}] {current}/{total} ({unit})")
+
+
+result = ParseHub().download_sync(
+    "https://example.com",
+    callback=ProgressTracker(),
+    callback_args=("extra_arg",),
+    callback_kwargs={"task_name": "demo"},
+)
+```
+
 ### Cookie 登录 & 代理
 
 部分平台的内容需要登录才能访问，通过 Cookie 即可解锁：
@@ -89,7 +108,7 @@ print(result)
 ```python
 from parsehub import ParseHub
 
-ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890",)
+ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890", )
 ```
 
 Cookie 支持多种格式传入：

{parsehub-2.0.3 → parsehub-2.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.0.3"
+version = "2.0.5"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/__init__.py

@@ -46,14 +46,18 @@ class ParseHub:
         *,
         callback: ProgressCallback = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
+        save_metadata: bool = False,
     ) -> DownloadResult:
         """下载
         :param url: 分享文案 / 分享链接
         :param path: 保存路径
         :param callback: 下载进度回调函数
         :param callback_args: 下载进度回调函数参数
+        :param callback_kwargs: 回调函数的关键字参数
         :param proxy: 代理
+        :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :return: DownloadResult
 
         Note:
@@ -68,7 +72,14 @@ class ParseHub:
                 - ``count``: 计数进度，用于多文件下载时报告已完成/总文件数
         """
         result = await self.parse(url)
-        return await result.download(path, callback=callback, callback_args=callback_args, proxy=proxy)
+        return await result.download(
+            path,
+            callback=callback,
+            callback_args=callback_args,
+            callback_kwargs=callback_kwargs,
+            proxy=proxy,
+            save_metadata=save_metadata,
+        )
 
     def download_sync(
         self,
@@ -76,7 +87,9 @@ class ParseHub:
         path: str | Path | None = None,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
+        save_metadata: bool = False,
     ) -> DownloadResult:
         """
         同步下载
@@ -84,7 +97,9 @@ class ParseHub:
         :param path: 下载路径
         :param callback: 进度回调函数
         :param callback_args: 进度回调函数参数
+        :param callback_kwargs: 回调函数的关键字参数
         :param proxy: 代理
+        :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :return: DownloadResult
 
         Note:
@@ -99,7 +114,15 @@ class ParseHub:
                 - ``count``: 计数进度，用于多文件下载时报告已完成/总文件数
         """
         return get_event_loop().run_until_complete(
-            self.download(url, path, callback=callback, callback_args=callback_args, proxy=proxy)
+            self.download(
+                url,
+                path,
+                callback=callback,
+                callback_args=callback_args,
+                callback_kwargs=callback_kwargs,
+                proxy=proxy,
+                save_metadata=save_metadata,
+            )
         )
 
     async def get_raw_url(self, url: str, proxy: str | None = None) -> str:
@@ -110,7 +133,7 @@ class ParseHub:
         """
         parser = self.get_parser(url)
         try:
-            return await parser(proxy=proxy).get_raw_url(url)
+            return await parser(proxy=proxy).get_raw_url(url, after_clean_parameters=True)
         except Exception as e:
             raise ParseError from e
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/base/base.py

@@ -25,6 +25,8 @@ class BaseParser(ABC):
     """匹配规则"""
     __reserved_parameters__: list[str] = []
     """要保留的参数, 例如翻页. 默认清除全部参数"""
+    __after_clean_parameters__: list[str] = []
+    """解析完成后需要清理的参数, 在解析完成前会保留这些参数, 优先级高于 __reserved_parameters__"""
     __redirect_keywords__: list[str] = []
     """如果链接包含其中之一, 则遵循重定向规则"""
 
@@ -61,9 +63,10 @@ class BaseParser(ABC):
         :param url: 分享文案 / 分享链接
         :return: 解析结果
         """
-        raw_url = await self.get_raw_url(url)
+        raw_url = await self.get_raw_url(url, after_clean_parameters=False)
         result = await self._do_parse(raw_url)
         result.platform = self.__platform__
+        result.raw_url = self._clean_params(raw_url, self.__after_clean_parameters__)
         return result
 
     @abstractmethod
@@ -73,10 +76,11 @@ class BaseParser(ABC):
         """
         raise NotImplementedError
 
-    async def get_raw_url(self, url: str) -> str:
+    async def get_raw_url(self, url: str, after_clean_parameters: bool = False) -> str:
         """
         清除链接中的参数
         :param url: 链接
+        :param after_clean_parameters: 是否执行后清理参数
         :return:
         """
         url = match_url(url)
@@ -101,7 +105,25 @@ class BaseParser(ABC):
         query_params = parse_qs(parsed_url.query)
 
         for i in query_params.copy().keys():
-            if i not in self.__reserved_parameters__:
-                del query_params[i]
+            is_reserved = i in self.__reserved_parameters__
+            is_after_clean = i in self.__after_clean_parameters__
+            keep = (is_reserved and not (after_clean_parameters and is_after_clean)) or (
+                is_after_clean and not after_clean_parameters
+            )
+            if not keep:
+                query_params.pop(i, None)
+
+        new_query = urlencode(query_params, doseq=True)
+        return parsed_url._replace(query=new_query).geturl()
+
+    @staticmethod
+    def _clean_params(url: str, params: list[str]) -> str:
+        """清除链接中的指定参数"""
+        if not params:
+            return url
+        parsed_url = urlparse(url)
+        query_params = parse_qs(parsed_url.query)
+        for p in params:
+            query_params.pop(p, None)
         new_query = urlencode(query_params, doseq=True)
         return parsed_url._replace(query=new_query).geturl()

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/base/ytdlp.py

@@ -39,7 +39,6 @@ class YtParser(BaseParser, register=False):
             dl=video_info,
             title=video_info.title,
             content=video_info.description,
-            raw_url=raw_url,
             video=VideoRef(
                 url=raw_url,
                 thumb_url=video_info.thumbnail,
@@ -115,11 +114,10 @@ class YtVideoParseResult(VideoParseResult):
         title,
         video=None,
         content=None,
-        raw_url=None,
     ):
         """dl: yt-dlp解析结果"""
         self.dl = dl
-        super().__init__(title=title, video=video, content=content, raw_url=raw_url)
+        super().__init__(title=title, video=video, content=content)
 
     async def _do_download(
         self,
@@ -127,9 +125,13 @@ class YtVideoParseResult(VideoParseResult):
         output_dir: str | Path,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         headers: dict | None = None,
     ) -> "DownloadResult":
+        if callback_kwargs is None:
+            callback_kwargs = {}
+
         paramss = self.dl.paramss.copy()
         if proxy:
             paramss["proxy"] = proxy
@@ -141,7 +143,7 @@ class YtVideoParseResult(VideoParseResult):
         #     paramss["format"] = "worstvideo* + worstaudio / worst"
 
         if callback:
-            await callback(0, 1, "count", *callback_args)
+            await callback(0, 1, "count", *callback_args, **callback_kwargs)
 
         await self.__download(paramss)
 
@@ -150,7 +152,7 @@ class YtVideoParseResult(VideoParseResult):
             raise DownloadError("下载失败 -1")
 
         if callback:
-            await callback(1, 1, "count", *callback_args)
+            await callback(1, 1, "count", *callback_args, **callback_kwargs)
 
         video_path = v[0]
         return DownloadResult(

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/bilibili.py

@@ -3,8 +3,6 @@ from pathlib import Path
 from typing import Union
 from urllib.parse import parse_qs, urlparse
 
-import httpx
-
 from ...config.config import GlobalConfig
 from ...provider_api.bilibili import BiliAPI, BiliDynamic
 from ...types import (
@@ -30,8 +28,8 @@ class BiliParse(YtParser):
     __redirect_keywords__ = ["b23.tv", "bili2233.cn"]
 
     async def _do_parse(self, raw_url: str) -> Union["YtVideoParseResult", "BiliVideoParseResult", ImageParseResult]:
-        if ourl := await self.is_dynamic(raw_url):
-            dynamic = await self.get_dynamic_info(ourl)
+        if await self.is_dynamic(raw_url):
+            dynamic = await self.get_dynamic_info(raw_url)
             content = self.hashtag_handler(dynamic.content)
             photos = []
             if dynamic.images:
@@ -44,7 +42,6 @@ class BiliParse(YtParser):
                 title=dynamic.title,
                 content=content,
                 photo=photos,
-                raw_url=ourl,
             )
         else:
             try:
@@ -69,18 +66,16 @@ class BiliParse(YtParser):
         else:
             return super().match(url)
 
-    async def get_raw_url(self, url: str) -> str:
+    async def get_raw_url(self, url: str, after_clean_parameters: bool = False) -> str:
         """获取原始链接"""
         if self._is_bvid(url):
             return f"https://www.bilibili.com/video/{url}"
         else:
-            return await super().get_raw_url(url)
+            return await super().get_raw_url(url, after_clean_parameters=after_clean_parameters)
 
-    async def is_dynamic(self, url) -> str | None:
+    @staticmethod
+    async def is_dynamic(url) -> str | None:
         """是动态"""
-        async with httpx.AsyncClient(proxy=self.proxy) as cli:
-            url = str((await cli.get(url, follow_redirects=True, timeout=30)).url)
-
         if re.search(r"\b\d{18,19}\b", url):
             return url
         return None
@@ -128,7 +123,6 @@ class BiliParse(YtParser):
         video_url = self.change_source(durl["backup_url"][0]) if durl.get("backup_url") else durl["url"]
         return BiliVideoParseResult(
             title=data["View"]["title"],
-            raw_url=url,
             content=f"P{p}: {part}" if part else "",
             video=VideoRef(
                 url=video_url,
@@ -143,7 +137,6 @@ class BiliParse(YtParser):
         result = await super()._do_parse(url)
         return YtVideoParseResult(
             title=result.title,
-            raw_url=result.raw_url,
             dl=result.dl,
             video=result.media,
         )
@@ -173,12 +166,18 @@ class BiliVideoParseResult(VideoParseResult):
         output_dir: str | Path,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         headers: dict | None = None,
     ) -> "DownloadResult":
         headers = {"referer": "https://www.bilibili.com", "User-Agent": GlobalConfig.ua}
         return await super()._do_download(
-            output_dir=output_dir, callback=callback, callback_args=callback_args, proxy=proxy, headers=headers
+            output_dir=output_dir,
+            callback=callback,
+            callback_args=callback_args,
+            callback_kwargs=callback_kwargs,
+            proxy=proxy,
+            headers=headers,
         )
 
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/coolapk.py

@@ -14,7 +14,6 @@ from ...types import (
     ProgressCallback,
     RichTextParseResult,
 )
-from ...utils.utils import clear_params
 from ..base.base import BaseParser
 
 
@@ -22,12 +21,11 @@ class CoolapkParser(BaseParser):
     __platform__ = Platform.COOLAPK
     __supported_type__ = ["图文"]
     __match__ = r"^(http(s)?://)www.coolapk.com/(feed|picture)/.*"
-    __reserved_parameters__ = ["shareKey", "s"]
+    __after_clean_parameters__ = ["shareKey", "s"]
 
     async def _do_parse(
         self, raw_url: str
     ) -> Union["CoolapkImageParseResult", "CoolapkRichTextParseResult", "CoolapkMultimediaParseResult"]:
-        raw_url_ = clear_params(raw_url, ["s", "shareKey"])
         try:
             coolapk = await Coolapk.parse(raw_url, proxy=self.proxy)
         except Exception as e:
@@ -38,20 +36,17 @@ class CoolapkParser(BaseParser):
                 title=coolapk.title,
                 media=media,
                 markdown_content=coolapk.markdown_content,
-                raw_url=raw_url_,
             )
         if any(isinstance(m, AniRef) for m in media):
             return CoolapkMultimediaParseResult(
                 title=coolapk.title,
                 media=media,
                 content=coolapk.text_content,
-                raw_url=raw_url_,
             )
         return CoolapkImageParseResult(
             title=coolapk.title,
             photo=media,
             content=coolapk.text_content,
-            raw_url=raw_url_,
         )
 
 
@@ -62,6 +57,7 @@ class CoolapkParseResult(ParseResult):
         output_dir: str | Path,
         callback: ProgressCallback = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         headers: dict = None,
     ) -> "DownloadResult":
@@ -72,7 +68,12 @@ class CoolapkParseResult(ParseResult):
             )
         }
         return await super()._do_download(
-            output_dir=output_dir, callback=callback, callback_args=callback_args, proxy=proxy, headers=headers
+            output_dir=output_dir,
+            callback=callback,
+            callback_args=callback_args,
+            callback_kwargs=callback_kwargs,
+            proxy=proxy,
+            headers=headers,
         )
 
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/douyin.py

@@ -30,9 +30,9 @@ class DouyinParser(BaseParser):
 
         match data.type:
             case DYType.VIDEO:
-                return await self.video_parse(raw_url, data)
+                return await self.video_parse(data)
             case DYType.IMAGE:
-                return await self.image_parse(raw_url, data)
+                return await self.image_parse(data)
 
     @staticmethod
     async def parse_api(url) -> "DYResult":
@@ -52,17 +52,15 @@ class DouyinParser(BaseParser):
         return DYResult.parse(url, response.json())
 
     @staticmethod
-    async def video_parse(url, result: "DYResult"):
+    async def video_parse(result: "DYResult"):
         return VideoParseResult(
-            raw_url=url,
             title=result.desc,
             video=result.video,
         )
 
     @staticmethod
-    async def image_parse(url, result: "DYResult"):
+    async def image_parse(result: "DYResult"):
         return ImageParseResult(
-            raw_url=url,
             title=result.desc,
             photo=result.image_list,
         )

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/kuaishou.py

@@ -25,7 +25,6 @@ class KuaiShouParser(BaseParser):
                     height=result.height,
                     width=result.width,
                 ),
-                raw_url=raw_url,
             )
 
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/pipix.py

@@ -27,10 +27,9 @@ class PipixParser(BaseParser):
                     height=ppx.video_height,
                     width=ppx.video_width,
                 ),
-                raw_url=raw_url,
             )
         else:
-            return ImageParseResult(title=ppx.content, photo=ppx.img_url, raw_url=raw_url)
+            return ImageParseResult(title=ppx.content, photo=ppx.img_url)
 
 
 __all__ = ["PipixParser"]

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/threads.py

@@ -19,7 +19,7 @@ class ThreadsParser(BaseParser):
                         media.append(VideoRef(url=m.url, thumb_url=m.thumb_url, width=m.width, height=m.height))
                     case ThreadsMediaType.IMAGE:
                         media.append(ImageRef(url=m.url, thumb_url=m.url, width=m.width, height=m.height))
-        return MultimediaParseResult(content=post.content, media=media, raw_url=raw_url)
+        return MultimediaParseResult(content=post.content, media=media)
 
 
 __all__ = ["ThreadsParser"]

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/tieba.py

@@ -17,9 +17,9 @@ class TieBaParser(BaseParser):
             raise ParseError("贴吧解析失败") from e
 
         if tb.video_url:
-            return VideoParseResult(title=tb.title, video=tb.video_url, raw_url=raw_url, content=tb.content)
+            return VideoParseResult(title=tb.title, video=tb.video_url, content=tb.content)
         else:
-            return ImageParseResult(title=tb.title, photo=tb.img_url, raw_url=raw_url, content=tb.content)
+            return ImageParseResult(title=tb.title, photo=tb.img_url, content=tb.content)
 
 
 __all__ = ["TieBaParser"]

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/twitter.py

@@ -21,8 +21,8 @@ class TwitterParser(BaseParser):
         tweet = await self._parse(raw_url)
         return await self.media_parse(raw_url, tweet)
 
-    async def get_raw_url(self, url: str) -> str:
-        url = await super().get_raw_url(url)
+    async def get_raw_url(self, url: str, after_clean_parameters: bool = False) -> str:
+        url = await super().get_raw_url(url, after_clean_parameters=after_clean_parameters)
         return str(urlunparse(urlparse(url)._replace(netloc="x.com")))
 
     async def _parse(self, url: str):
@@ -63,7 +63,7 @@ class TwitterParser(BaseParser):
                 case TwitterAni():
                     path = AniRef(url=m.url, ext="mp4", height=m.height, width=m.width, thumb_url=m.thumb_url)
             media.append(path)
-        return MultimediaParseResult(content=tweet.full_text, media=media, raw_url=url)
+        return MultimediaParseResult(content=tweet.full_text, media=media)
 
 
 __all__ = ["TwitterParser"]

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/weibo.py

@@ -29,7 +29,6 @@ class WeiboParser(BaseParser):
             if data.page_info.object_type == MediaType.VIDEO:
                 return VideoParseResult(
                     content=text,
-                    raw_url=raw_url,
                     video=VideoRef(
                         url=data.page_info.media_info.playback.url,
                         thumb_url=data.page_info.page_pic,
@@ -45,7 +44,7 @@ class WeiboParser(BaseParser):
             or (data.mix_media_info and data.mix_media_info.items)
         )
         if not media_info:
-            return MultimediaParseResult(content=text, raw_url=raw_url, media=[])
+            return MultimediaParseResult(content=text, media=[])
 
         for i in media_info:
             match i.type:
@@ -70,8 +69,8 @@ class WeiboParser(BaseParser):
                 case _:
                     media.append(ImageRef(url=i.media_url, thumb_url=i.thumb_url, width=i.width, height=i.height))
         if all((isinstance(m, ImageRef) or isinstance(m, LivePhotoRef)) for m in media):
-            return ImageParseResult(content=text, raw_url=raw_url, photo=media)
-        return MultimediaParseResult(content=text, raw_url=raw_url, media=media)
+            return ImageParseResult(content=text, photo=media)
+        return MultimediaParseResult(content=text, media=media)
 
     def f_text(self, text: str) -> str:
         # text = re.sub(r'<a  href="https://video.weibo.com.*?>.*的微博视频.*</a>', "", text)

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/weixin.py

@@ -14,7 +14,6 @@ class WXParser(BaseParser):
             title=wx.title,
             media=[ImageRef(url=i) for i in wx.imgs],
             markdown_content=wx.markdown_content,
-            raw_url=raw_url,
         )
 
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/xhs.py

@@ -14,7 +14,6 @@ from ...types import (
     VideoParseResult,
     VideoRef,
 )
-from ...utils.utils import clear_params
 from ..base import BaseParser
 
 
@@ -23,15 +22,14 @@ class XHSParser(BaseParser):
     __supported_type__ = ["视频", "图文"]
     __match__ = r"^(http(s)?://)?.+(xiaohongshu|xhslink).com/.+"
     __redirect_keywords__ = ["xhslink", "item"]
-    __reserved_parameters__ = ["xsec_token"]
+    __after_clean_parameters__ = ["xsec_token"]
 
     async def _do_parse(self, raw_url: str) -> Union["VideoParseResult", "ImageParseResult", "MultimediaParseResult"]:
-        raw_url_ = clear_params(raw_url, "xsec_token")
         xhs = XHSAPI(proxy=self.proxy)
         result = await xhs.extract(raw_url)
 
         desc = self.hashtag_handler(result.desc)
-        k = {"title": result.title, "content": desc, "raw_url": raw_url_}
+        k = {"title": result.title, "content": desc, "raw_url": raw_url}
         match result.type:
             case XHSPostType.VIDEO:
                 v: XHSMedia = result.media[0]

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/xiaoheihe.py

@@ -31,7 +31,7 @@ class XiaoHeiHeParser(BaseParser):
                     return ImageParseResult(photo=media, **v)
                 return MultimediaParseResult(media=media, **v)
             case XiaoHeiHePostType.ARTICLE:
-                return RichTextParseResult(title=xhh.title, media=media, markdown_content=xhh.content, raw_url=raw_url)
+                return RichTextParseResult(title=xhh.title, media=media, markdown_content=xhh.content)
 
     @staticmethod
     def __parse_media(xhh: XiaoHeiHePost):

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/parsers/parser/zuiyou.py

@@ -19,7 +19,6 @@ class ZuiYouParser(BaseParser):
                 else ImageRef(url=i.url, thumb_url=i.thumb_url)
                 for i in zy.media
             ],
-            raw_url=raw_url,
         )
 
 

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/provider_api/weibo.py

@@ -294,4 +294,4 @@ class WeiboContent:
 
 
 if __name__ == "__main__":
-    print(asyncio.run(WeiboAPI().parse("https://weibo.com/3208333150/Ow0iEbEX0")))
+    print(asyncio.run(WeiboAPI().parse("https://weibo.com/6576374129/Qv0n8sXum")))

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/types/callback.py

@@ -6,7 +6,7 @@ ProgressUnit = Literal["bytes", "count"]
 class ProgressCallback(Protocol):
     """下载进度回调: (current, total, unit, *args) -> None"""
 
-    async def __call__(self, current: int, total: int, unit: ProgressUnit, *args) -> None:
+    async def __call__(self, current: int, total: int, unit: ProgressUnit, *args, **kwargs) -> None:
         """
         下载进度回调
         Args:
@@ -16,6 +16,7 @@ class ProgressCallback(Protocol):
                 - ``bytes``: 字节进度，用于单文件下载时报告已下载/总字节数
                 - ``count``: 计数进度，用于多文件下载时报告已完成/总文件数
             *args: 自定义参数
+            **kwargs: 自定义关键字参数
 
         Returns:
             None

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/types/result.py

@@ -29,7 +29,6 @@ class ParseResult(ABC):  # noqa: B024
 
     def __init__(
         self,
-        raw_url: str,
         title: str = "",
         content: str = "",
         media: list[AnyMediaRef] | AnyMediaRef | None = None,
@@ -39,13 +38,12 @@ class ParseResult(ABC):  # noqa: B024
         :param title: 标题
         :param media: 媒体下载链接
         :param content: 正文 (纯文本)
-        :param raw_url: 原始帖子链接
         :param platform: 平台
         """
+        self.raw_url = None
         self.title = (title or "").strip()
         self.content = (content or "").strip()
         self.media = media
-        self.raw_url = raw_url
         self.platform = platform
 
     def __repr__(self):
@@ -79,6 +77,7 @@ class ParseResult(ABC):  # noqa: B024
         output_dir: str | Path,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         headers: dict | None = None,
     ) -> "DownloadResult":
@@ -87,6 +86,7 @@ class ParseResult(ABC):  # noqa: B024
         :param output_dir: 输出的子目录
         :param callback: 下载进度回调函数
         :param callback_args: 回调函数的参数
+        :param callback_kwargs: 回调函数的关键字参数
         :param proxy: 代理
         :param headers: 请求头
         :return: DownloadResult
@@ -99,13 +99,15 @@ class ParseResult(ABC):  # noqa: B024
         for i, media in enumerate(media_list):
             dl_progress = None
             dl_progress_args = ()
+            dl_progress_kwargs = {}
             if callback and is_single:
 
-                async def _byte_callback(current, total, *args):
-                    await callback(current, total, "bytes", *args)
+                async def _byte_callback(current, total, *args, **kwargs):
+                    await callback(current, total, "bytes", *args, **kwargs)
 
                 dl_progress = _byte_callback
                 dl_progress_args = callback_args
+                dl_progress_kwargs = callback_kwargs
 
             try:
                 f = await download(
@@ -115,6 +117,7 @@ class ParseResult(ABC):  # noqa: B024
                     proxies=proxy,
                     progress=dl_progress,
                     progress_args=dl_progress_args,
+                    progress_kwargs=dl_progress_kwargs,
                 )
             except Exception as e:
                 shutil.rmtree(output_dir, ignore_errors=True)
@@ -162,6 +165,7 @@ class ParseResult(ABC):  # noqa: B024
         *,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         save_metadata: bool = False,
     ) -> "DownloadResult":
@@ -169,6 +173,7 @@ class ParseResult(ABC):  # noqa: B024
         :param path: 保存路径
         :param callback: 下载进度回调函数
         :param callback_args: 下载进度回调函数参数
+        :param callback_kwargs: 回调函数的关键字参数
         :param proxy: 代理
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :return: DownloadResult
@@ -176,7 +181,7 @@ class ParseResult(ABC):  # noqa: B024
         Note:
             下载进度回调函数签名::
 
-                async def callback(current: int, total: int, unit: Literal['bytes', 'count'], *args) -> None
+                async def callback(current: int, total: int, unit: Literal['bytes', 'count'], *args, **kwargs) -> None
 
             - current: 当前进度值
             - total: 总进度值
@@ -201,7 +206,11 @@ class ParseResult(ABC):  # noqa: B024
 
         try:
             return await self._do_download(
-                output_dir=output_dir, callback=callback, callback_args=callback_args, proxy=proxy
+                output_dir=output_dir,
+                callback=callback,
+                callback_args=callback_args,
+                callback_kwargs=callback_kwargs,
+                proxy=proxy,
             )
         except Exception as e:
             shutil.rmtree(output_dir, ignore_errors=True)
@@ -213,6 +222,7 @@ class ParseResult(ABC):  # noqa: B024
         *,
         callback: ProgressCallback | None = None,
         callback_args: tuple = (),
+        callback_kwargs: dict | None = None,
         proxy: str | None = None,
         save_metadata: bool = False,
     ) -> "DownloadResult":
@@ -220,6 +230,7 @@ class ParseResult(ABC):  # noqa: B024
         :param path: 保存路径
         :param callback: 下载进度回调函数
         :param callback_args: 下载进度回调函数参数
+        :param callback_kwargs: 回调函数的关键字参数
         :param proxy: 代理
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :return: DownloadResult
@@ -237,7 +248,12 @@ class ParseResult(ABC):  # noqa: B024
         """
         return get_event_loop().run_until_complete(
             self.download(
-                path, callback=callback, callback_args=callback_args, proxy=proxy, save_metadata=save_metadata
+                path,
+                callback=callback,
+                callback_args=callback_args,
+                callback_kwargs=callback_kwargs,
+                proxy=proxy,
+                save_metadata=save_metadata,
             )
         )
 
@@ -249,7 +265,6 @@ class VideoParseResult(ParseResult):
 
     def __init__(
         self,
-        raw_url: str,
         title: str = "",
         video: str | VideoRef | None = None,
         content: str = "",
@@ -259,7 +274,6 @@ class VideoParseResult(ParseResult):
             title=title,
             media=video,
             content=content,
-            raw_url=raw_url,
         )
 
 
@@ -270,14 +284,13 @@ class ImageParseResult(ParseResult):
 
     def __init__(
         self,
-        raw_url: str,
         title: str = "",
         photo: list[str | ImageRef | LivePhotoRef] | None = None,
         content: str = "",
     ):
         if photo:
             photo = [ImageRef(url=p) if isinstance(p, str) else p for p in photo]
-        super().__init__(title=title, media=photo, content=content, raw_url=raw_url)
+        super().__init__(title=title, media=photo, content=content)
 
 
 class MultimediaParseResult(ParseResult):
@@ -287,12 +300,11 @@ class MultimediaParseResult(ParseResult):
 
     def __init__(
         self,
-        raw_url: str,
         title: str = "",
         media: list[AnyMediaRef] | None = None,
         content: str = "",
     ):
-        super().__init__(title=title, media=media, content=content, raw_url=raw_url)
+        super().__init__(title=title, media=media, content=content)
 
 
 class RichTextParseResult(ParseResult):
@@ -302,20 +314,17 @@ class RichTextParseResult(ParseResult):
 
     def __init__(
         self,
-        raw_url: str,
         title: str = "",
         media: list[AnyMediaRef] | None = None,
         markdown_content: str = "",
     ):
         """
-
         :param title: 标题
         :param media: 文章中的媒体
         :param markdown_content: markdown 格式正文
-        :param raw_url: 原始 URL
         """
         self.markdown_content = markdown_content
-        super().__init__(title=title, media=media, content=self.plaintext_content, raw_url=raw_url)
+        super().__init__(title=title, media=media, content=self.plaintext_content)
 
     def __repr__(self):
         media_count = f"[{len(self.media if isinstance(self.media, list) else [self.media])}]" if self.media else None

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/utils/downloader.py

@@ -3,6 +3,7 @@ import os
 import re
 from collections.abc import Callable
 from pathlib import Path
+from typing import Literal
 from urllib.parse import unquote, urlparse
 
 import aiofiles
@@ -17,6 +18,7 @@ async def download(
     proxies: httpx.Proxy | None = None,
     progress: Callable | None = None,
     progress_args: tuple = (),
+    progress_kwargs: dict | None = None,
     max_retries: int = 3,
     chunk_size: int = 8192,
 ) -> str:
@@ -26,7 +28,8 @@ async def download(
     :param headers: 请求头
     :param proxies: 代理
     :param progress: 下载进度回调函数
-    :param progress_args: 下载进度回调函数参数
+    :param progress_args: 下载进度回调函数的参数
+    :param progress_kwargs: 下载进度回调函数的关键字参数
     :param max_retries: 最大重试次数
     :param chunk_size: 分块大小
     :return: 文件路径
@@ -78,7 +81,7 @@ async def download(
 
                     current = resume_pos if is_resumed else 0
 
-                    file_mode = "ab" if is_resumed else "wb"
+                    file_mode: Literal["ab", "wb"] = "ab" if is_resumed else "wb"
 
                     async with aiofiles.open(file=resolved_path, mode=file_mode) as f:
                         async for chunk in r.aiter_bytes(chunk_size=chunk_size):
@@ -86,7 +89,9 @@ async def download(
                                 await f.write(chunk)
                                 current += len(chunk)
                                 if progress:
-                                    await progress(current, total_size, *progress_args)
+                                    if progress_kwargs is None:
+                                        progress_kwargs = {}
+                                    await progress(current, total_size, *progress_args, **progress_kwargs)
 
                     # 完整性校验
                     if 0 < total_size != current:

{parsehub-2.0.3 → parsehub-2.0.5}/src/parsehub/utils/utils.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import re
-from urllib.parse import parse_qs, urlparse
 
 from urlextract import URLExtract
 
@@ -15,7 +14,7 @@ def get_event_loop():
     return event_loop
 
 
-url_extractor = URLExtract()
+_url_extractor = URLExtract()
 
 
 def match_url(text: str) -> str:
@@ -23,7 +22,7 @@ def match_url(text: str) -> str:
     if not text:
         return ""
     text = re.sub(r"(https?://)", r" \1", text)  # 协议前面增加空格, 方便提取
-    url = url_extractor.find_urls(text, only_unique=True)
+    url = _url_extractor.find_urls(text, only_unique=True)
     return url[0] if url else ""
 
 
@@ -35,23 +34,6 @@ def cookie_ellipsis(cookie: dict) -> str:
     return f"{text[:c]}......{text[-c:]}"
 
 
-def clear_params(url: str, param: str | list[str]) -> str:
-    """
-    删除链接指定参数
-    :param url: 链接
-    :param param: 参数
-    :return:
-    """
-    params = param if isinstance(param, list) else [param]
-    parsed_url = urlparse(url)
-    query_params = parse_qs(parsed_url.query)
-    for i in params.copy():
-        if i in query_params:
-            del query_params[i]
-    new_query = "&".join([f"{k}={v[0]}" for k, v in query_params.items()])
-    return parsed_url._replace(query=new_query).geturl()
-
-
 def normalize_cookie(v):
     if v is None or isinstance(v, dict):
         return v

{parsehub-2.0.3 → parsehub-2.0.5/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.3
+Version: 2.0.5
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -119,6 +119,25 @@ print(result)
 
 ## 🔑 高级用法
 
+### 下载进度回调
+
+```python
+from parsehub import ParseHub
+
+
+class ProgressTracker:
+    async def __call__(self, current: int, total: int, unit: str, *args, task_name: str = "", **kwargs):
+        print(f"[{task_name}] {current}/{total} ({unit})")
+
+
+result = ParseHub().download_sync(
+    "https://example.com",
+    callback=ProgressTracker(),
+    callback_args=("extra_arg",),
+    callback_kwargs={"task_name": "demo"},
+)
+```
+
 ### Cookie 登录 & 代理
 
 部分平台的内容需要登录才能访问，通过 Cookie 即可解锁：
@@ -126,7 +145,7 @@ print(result)
 ```python
 from parsehub import ParseHub
 
-ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890",)
+ph = ParseHub(cookie="key1=value1; key2=value2", proxy="http://127.0.0.1:7890", )
 ```
 
 Cookie 支持多种格式传入：