parsehub 2.3.7__tar.gz → 2.3.9__tar.gz

{parsehub-2.3.7/src/parsehub.egg-info → parsehub-2.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.3.7
+Version: 2.3.9
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT

{parsehub-2.3.7 → parsehub-2.3.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.3.7"
+version = "2.3.9"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/__init__.py

@@ -61,6 +61,7 @@ class ParseHub:
         parse_cookie: str | dict | None = None,
         save_metadata: bool = False,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> DownloadResult:
         """下载
         :param url: 分享文案 / 分享链接
@@ -73,6 +74,7 @@ class ParseHub:
         :param parse_cookie: 解析 cookie
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :param connections: 多线程下载连接数, 默认为 4
+        :param concurrency: 多文件并发下载数, 默认为 4
         :return: DownloadResult
 
         Note:
@@ -95,6 +97,7 @@ class ParseHub:
             proxy=proxy,
             save_metadata=save_metadata,
             connections=connections,
+            concurrency=concurrency,
         )
 
     def download_sync(
@@ -109,6 +112,7 @@ class ParseHub:
         parse_cookie: str | dict | None = None,
         save_metadata: bool = False,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> DownloadResult:
         """
         同步下载
@@ -122,6 +126,7 @@ class ParseHub:
         :param parse_cookie: 解析 cookie
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :param connections: 多线程下载连接数, 默认为 4
+        :param concurrency: 多文件并发下载数, 默认为 4
         :return: DownloadResult
 
         Note:
@@ -147,6 +152,7 @@ class ParseHub:
                 parse_cookie=parse_cookie,
                 save_metadata=save_metadata,
                 connections=connections,
+                concurrency=concurrency,
             )
         )
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/cli.py

@@ -117,6 +117,7 @@ def _build_parser(prog: str) -> argparse.ArgumentParser:
     download_parser.add_argument("-q", "--quiet", action="store_true", help="不输出状态和进度信息")
     download_parser.add_argument("--no-progress", action="store_true", help="不显示下载进度")
     download_parser.add_argument("--connections", type=int, default=4, help="单文件分片下载连接数，设为 1 可禁用分片")
+    download_parser.add_argument("--concurrency", type=int, default=4, help="多文件并发下载数，设为 1 即串行")
     _add_json_options(download_parser)
     download_parser.set_defaults(func=_cmd_download)
 
@@ -258,6 +259,7 @@ def _cmd_download(args: argparse.Namespace) -> int:
         parse_cookie=parse_cookie,
         save_metadata=args.save_metadata,
         connections=args.connections,
+        concurrency=args.concurrency,
     )
     reporter.finish()
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/base/ytdlp.py

@@ -477,6 +477,7 @@ class YtVideoParseResult(VideoParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         if callback_kwargs is None:
             callback_kwargs = {}

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/bilibili.py

@@ -191,6 +191,7 @@ class BiliVideoParseResult(VideoParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> DownloadResult:
         headers = {"referer": "https://www.bilibili.com", "User-Agent": UA}
         return await super()._do_download(
@@ -201,6 +202,7 @@ class BiliVideoParseResult(VideoParseResult):
             proxy=proxy,
             headers=headers,
             connections=connections,
+            concurrency=concurrency,
         )
 
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/coolapk.py

@@ -70,6 +70,7 @@ class CoolapkParseResult(ParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         headers = {
             "Accept": (
@@ -85,6 +86,7 @@ class CoolapkParseResult(ParseResult):
             proxy=proxy,
             headers=headers,
             connections=connections,
+            concurrency=concurrency,
         )
 
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/douban.py

@@ -78,6 +78,7 @@ class DoubanParseResult(ParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         headers = {"User-Agent": UA, "Referer": IMAGE_REFERER}
         return await super()._do_download(
@@ -88,6 +89,7 @@ class DoubanParseResult(ParseResult):
             proxy=proxy,
             headers=headers,
             connections=connections,
+            concurrency=concurrency,
         )
 
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/douyin.py

@@ -88,6 +88,7 @@ class DouyinParseResult(ParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         headers = {
             "Referer": "https://www.douyin.com/",
@@ -100,6 +101,7 @@ class DouyinParseResult(ParseResult):
             proxy=proxy,
             headers=headers,
             connections=connections,
+            concurrency=concurrency,
         )
 
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/tiktok.py

@@ -70,6 +70,7 @@ class TikTokVideoParseResult(VideoParseResult):
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         headers = {
             "Referer": "https://www.tiktok.com/",
@@ -82,6 +83,7 @@ class TikTokVideoParseResult(VideoParseResult):
             proxy=proxy,
             headers=headers,
             connections=connections,
+            concurrency=concurrency,
         )
 
 

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/parsers/parser/xiaoheihe.py

@@ -54,7 +54,23 @@ class XiaoHeiHeParser(BaseParser):
         images: list[ImageRef | AniRef] = []
         for media in xhh.media or []:
             if media.type == XiaoHeiHeMediaType.IMAGE:
-                images.append(ImageRef(url=media.url, width=media.width or 0, height=media.height or 0))
+                images.append(
+                    ImageRef(
+                        url=media.url,
+                        thumb_url=media.thumb_url,
+                        ext=media.ext or "jpg",
+                        width=media.width or 0,
+                        height=media.height or 0,
+                    )
+                )
             else:
-                images.append(AniRef(url=media.url, width=media.width or 0, height=media.height or 0))
+                images.append(
+                    AniRef(
+                        url=media.url,
+                        thumb_url=media.thumb_url,
+                        ext=media.ext or "gif",
+                        width=media.width or 0,
+                        height=media.height or 0,
+                    )
+                )
         return images

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/provider_api/xiaoheihe.py

@@ -23,6 +23,11 @@ from cryptography.hazmat.primitives.ciphers.modes import CBC, ECB
 from markdownify import MarkdownConverter
 
 
+def original_image_url(url: str) -> str:
+    path = url.split("?", 1)[0]
+    return re.sub(r"/thumb\.(\w+)$", r".\1", path)
+
+
 class XiaoHeiHePostType(Enum):
     VIDEO = "video"
     IMAGE = "image"  # 图文, 单张或多张图片 + 文案
@@ -40,6 +45,7 @@ class XiaoHeiHeMedia:
     type: XiaoHeiHeMediaType
     url: str
     thumb_url: str | None = None
+    ext: str | None = None
     height: int | None = None
     width: int | None = None
 
@@ -89,11 +95,14 @@ class XiaoHeiHeAPI:
             for image in text_list[1:]:
                 if image["type"] == "img":
                     media_type = XiaoHeiHeMediaType.GIF if ".gif" in image["url"] else XiaoHeiHeMediaType.IMAGE
+                    url = original_image_url(image["url"])
+                    ext_match = re.search(r"\.([^./]+)$", url)
                     images.append(
                         XiaoHeiHeMedia(
                             type=media_type,
-                            url=image["url"],
+                            url=url,
                             thumb_url=image["url"],
+                            ext=ext_match.group(1) if ext_match else None,
                             height=int(float(image.get("height", 0))),
                             width=int(float(image.get("width", 0))),
                         )
@@ -134,7 +143,7 @@ class XiaoHeiHeAPI:
             "owner_only": "1",
             **sig_params,
         }
-        cookies = {"x_xhh_tokenid": await SecuritySm.get_d_id()}
+        cookies = {"x_xhh_tokenid": await SecuritySm.get_d_id(self.proxy)}
         async with httpx.AsyncClient(proxy=self.proxy) as cli:
             result = await cli.get(self.api_url + "/bbs/app/link/tree", params=params, cookies=cookies)
             result.raise_for_status()
@@ -355,6 +364,8 @@ class XHHConverter(MarkdownConverter):
     def convert_img(self, el, _, parent_tags):
         alt = el.attrs.get("alt", None) or ""
         src = el.attrs.get("data-original", None) or ""
+        if src:
+            src = original_image_url(src)
         title = el.attrs.get("title", None) or ""
         title_part = ' "{}"'.format(title.replace('"', r"\"")) if title else ""
         options = cast(dict[str, Any], getattr(self, "options"))  # noqa: B009
@@ -504,7 +515,7 @@ class SecuritySm:
         return v + smsk_web + "0"
 
     @classmethod
-    async def get_d_id(cls):
+    async def get_d_id(cls, proxy: str | None = None):
         uid = str(uuid.uuid4()).encode("utf-8")
         priId = hashlib.md5(uid).hexdigest()[0:16]
         encrypted_uid = cls.PK.encrypt(uid, padding.PKCS1v15())
@@ -531,7 +542,7 @@ class SecuritySm:
         des_target["tn"] = hashlib.md5(cls.get_tn(des_target).encode()).hexdigest()
 
         des_result = cls._AES(cls.GZIP(cls._DES(des_target)), priId.encode("utf-8"))
-        async with httpx.AsyncClient() as client:
+        async with httpx.AsyncClient(proxy=proxy) as client:
             response = await client.post(
                 cls.DEVICES_INFO_URL,
                 json={

{parsehub-2.3.7 → parsehub-2.3.9}/src/parsehub/types/result.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import shutil
 import time
@@ -89,6 +90,7 @@ class ParseResult(ABC):  # noqa: B024
         proxy: str | None = None,
         headers: dict | None = None,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         """
         执行下载
@@ -99,48 +101,34 @@ class ParseResult(ABC):  # noqa: B024
         :param proxy: 代理
         :param headers: 请求头
         :param connections: 多线程下载连接数, 默认为 4
+        :param concurrency: 多文件并发下载数, 默认为 4
         :return: DownloadResult
         """
         if self.media is None:
             return DownloadResult(output_dir=output_dir, media=[])
         media_list = list(self.media) if isinstance(self.media, Sequence) else [self.media]
         is_single = not isinstance(self.media, Sequence)
-
-        result_list: list[AnyMediaFile] = []
-
-        for i, media in enumerate(media_list):
-            dl_progress = None
-            dl_progress_args = ()
-            dl_progress_kwargs: dict = {}
-            if callback and is_single:
-
-                async def _byte_callback(current: int, total: int, *args: Any, **kwargs: Any) -> None:
-                    await callback(current, total, "bytes", *args, **kwargs)
-
-                dl_progress = _byte_callback
-                dl_progress_args = callback_args
-                dl_progress_kwargs = callback_kwargs or {}
-
-            index = i + 1
-
+        concurrency = max(1, concurrency)
+
+        async def _download_media(
+            media: AnyMediaRef,
+            stem: str,
+            progress: ProgressCallback | None = None,
+            progress_args: tuple = (),
+            progress_kwargs: dict | None = None,
+        ) -> AnyMediaFile:
             try:
-                save_path = (
-                    output_dir.joinpath(f"{self.name}.{media.ext}")
-                    if is_single
-                    else output_dir.joinpath(f"{index:03d}_{self.name}.{media.ext}")
-                )
                 f = await download(
                     media.url,
-                    save_path,
+                    output_dir.joinpath(f"{stem}.{media.ext}"),
                     headers=headers,
                     proxy=proxy,
-                    progress=dl_progress,
-                    progress_args=dl_progress_args,
-                    progress_kwargs=dl_progress_kwargs,
+                    progress=progress,
+                    progress_args=progress_args,
+                    progress_kwargs=progress_kwargs,
                     connections=connections,
                 )
             except Exception as e:
-                shutil.rmtree(output_dir, ignore_errors=True)
                 raise DownloadError(f"下载失败: {e}") from e
 
             mf: AnyMediaFile
@@ -155,36 +143,74 @@ class ParseResult(ABC):  # noqa: B024
                     mf = LivePhotoFile(path=f, width=media.width, height=media.height, duration=media.duration)
                     if media.video_url:
                         try:
-                            save_path = (
-                                output_dir.joinpath(f"{self.name}_video.{media.video_ext}")
-                                if is_single
-                                else output_dir.joinpath(f"{index:03d}_{self.name}_video.{media.video_ext}")
-                            )
                             vf = await download(
                                 media.video_url,
-                                save_path,
+                                output_dir.joinpath(f"{stem}_video.{media.video_ext}"),
                                 headers=headers,
                                 proxy=proxy,
                                 connections=connections,
                             )
                         except Exception as e:
-                            shutil.rmtree(output_dir, ignore_errors=True)
                             raise DownloadError(f"LivePhoto 视频下载失败: {e}") from e
                         else:
                             mf.video_path = vf
 
-            result_list.append(mf)
+            return mf
+
+        if is_single:
+            media = media_list[0]
+            dl_progress = None
+            dl_progress_args = ()
+            dl_progress_kwargs: dict = {}
+            if callback:
+
+                async def _byte_callback(current: int, total: int, *args: Any, **kwargs: Any) -> None:
+                    await callback(current, total, "bytes", *args, **kwargs)
 
-            if callback and not is_single:
-                await callback(
-                    len(result_list),
-                    len(media_list),
-                    "count",
-                    *callback_args,
+                dl_progress = _byte_callback
+                dl_progress_args = callback_args
+                dl_progress_kwargs = callback_kwargs or {}
+
+            try:
+                mf = await _download_media(
+                    media,
+                    self.name,
+                    progress=dl_progress,
+                    progress_args=dl_progress_args,
+                    progress_kwargs=dl_progress_kwargs,
                 )
+            except DownloadError:
+                shutil.rmtree(output_dir, ignore_errors=True)
+                raise
+            return DownloadResult(mf, output_dir)
+
+        n = len(media_list)
+        results: list[AnyMediaFile | None] = [None] * n
+        semaphore = asyncio.Semaphore(concurrency)
+        completed = 0
+
+        async def _download_one(i: int, media: AnyMediaRef) -> AnyMediaFile:
+            nonlocal completed
+            async with semaphore:
+                mf = await _download_media(media, f"{i + 1:03d}_{self.name}")
+                results[i] = mf
+                completed += 1
+                if callback:
+                    await callback(completed, n, "count", *callback_args)
+                return mf
+
+        tasks = [asyncio.create_task(_download_one(i, media)) for i, media in enumerate(media_list)]
+        try:
+            await asyncio.gather(*tasks)
+        except BaseException:
+            for task in tasks:
+                task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            shutil.rmtree(output_dir, ignore_errors=True)
+            raise
 
-        result_media = result_list[0] if is_single else result_list
-        return DownloadResult(result_media, output_dir)
+        result_list = [media for media in results if media is not None]
+        return DownloadResult(result_list, output_dir)
 
     async def download(
         self,
@@ -196,6 +222,7 @@ class ParseResult(ABC):  # noqa: B024
         proxy: str | None = None,
         save_metadata: bool = False,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         """
         :param path: 保存路径
@@ -205,6 +232,7 @@ class ParseResult(ABC):  # noqa: B024
         :param proxy: 代理
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :param connections: 多线程下载连接数, 默认为 4
+        :param concurrency: 多文件并发下载数, 默认为 4
         :return: DownloadResult
 
         Note:
@@ -238,6 +266,7 @@ class ParseResult(ABC):  # noqa: B024
                 callback_kwargs=callback_kwargs,
                 proxy=proxy,
                 connections=connections,
+                concurrency=concurrency,
             )
         except Exception as e:
             shutil.rmtree(output_dir, ignore_errors=True)
@@ -253,6 +282,7 @@ class ParseResult(ABC):  # noqa: B024
         proxy: str | None = None,
         save_metadata: bool = False,
         connections: int = 4,
+        concurrency: int = 4,
     ) -> "DownloadResult":
         """
         :param path: 保存路径
@@ -262,6 +292,7 @@ class ParseResult(ABC):  # noqa: B024
         :param proxy: 代理
         :param save_metadata: 保存解析结果为 metadata.json, 默认为 False
         :param connections: 多线程下载连接数, 默认为 4
+        :param concurrency: 多文件并发下载数, 默认为 4
         :return: DownloadResult
 
         Note:
@@ -284,6 +315,7 @@ class ParseResult(ABC):  # noqa: B024
                 proxy=proxy,
                 save_metadata=save_metadata,
                 connections=connections,
+                concurrency=concurrency,
             )
         )
 

{parsehub-2.3.7 → parsehub-2.3.9/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.3.7
+Version: 2.3.9
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT

{parsehub-2.3.7 → parsehub-2.3.9}/test/test_cli.py

@@ -62,6 +62,7 @@ class FakeParseHub:
         parse_cookie=None,
         save_metadata=False,
         connections=4,
+        concurrency=4,
     ):
         self.download_calls.append(
             {
@@ -75,6 +76,7 @@ class FakeParseHub:
                 "parse_cookie": parse_cookie,
                 "save_metadata": save_metadata,
                 "connections": connections,
+                "concurrency": concurrency,
             }
         )
         if callback: