parsehub 2.0.40__tar.gz → 2.0.41__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parsehub-2.0.40/src/parsehub.egg-info → parsehub-2.0.41}/PKG-INFO +20 -19
- {parsehub-2.0.40 → parsehub-2.0.41}/README.md +18 -18
- {parsehub-2.0.40 → parsehub-2.0.41}/pyproject.toml +2 -1
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/douyin.py +32 -11
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/douyin.py +449 -18
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/utils/helpers.py +3 -1
- {parsehub-2.0.40 → parsehub-2.0.41/src/parsehub.egg-info}/PKG-INFO +20 -19
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub.egg-info/requires.txt +1 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/test/test_core_offline.py +97 -1
- {parsehub-2.0.40 → parsehub-2.0.41}/LICENSE +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/setup.cfg +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/cli.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/cli_config.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/config/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/config/config.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/errors.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/base/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/base/base.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/base/ytdlp.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/bilibili.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/coolapk.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/facebook.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/instagram.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/kuaishou.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/pipix.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/snapchat.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/threads.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/tieba.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/tiktok.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/twitter.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/weibo.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/weixin.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/xhs.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/youtube.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/parsers/parser/zuiyou.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/bilibili.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/coolapk.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/instagram.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/kuaishou.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/pipix.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/threads.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/tieba.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/tiktok.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/twitter.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/weibo.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/weixin.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/xhs.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/xiaoheihe.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/provider_api/zuiyou.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/__init__.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/callback.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/media_file.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/media_ref.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/platform.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/post.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/types/result.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/utils/downloader.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub/utils/media_info.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub.egg-info/SOURCES.txt +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub.egg-info/dependency_links.txt +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub.egg-info/entry_points.txt +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/src/parsehub.egg-info/top_level.txt +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/test/test_cli.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/test/test_cli_config.py +0 -0
- {parsehub-2.0.40 → parsehub-2.0.41}/test/test_downloader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.41
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -34,6 +34,7 @@ Requires-Dist: python-slugify[unidecode]>=8.0.4
|
|
|
34
34
|
Requires-Dist: opencv-python-headless>=4.13.0.92
|
|
35
35
|
Requires-Dist: cryptography>=49.0.0
|
|
36
36
|
Requires-Dist: gmssl>=3.2.2
|
|
37
|
+
Requires-Dist: SignerPy>=0.12.0
|
|
37
38
|
Provides-Extra: cli
|
|
38
39
|
Requires-Dist: argcomplete>=3.6.3; extra == "cli"
|
|
39
40
|
Requires-Dist: platformdirs>=4.5.1; extra == "cli"
|
|
@@ -69,26 +70,26 @@ Dynamic: license-file
|
|
|
69
70
|
|
|
70
71
|
## 🌐 支持平台
|
|
71
72
|
|
|
72
|
-
| 平台 | 视频 | 图文 | 其他
|
|
73
|
-
|
|
74
|
-
| **Twitter / X** | ✅ | ✅ |
|
|
75
|
-
| **Instagram** | ✅ | ✅ |
|
|
73
|
+
| 平台 | 视频 | 图文 | 其他 |
|
|
74
|
+
|-----------------|:--:|:-:|------|
|
|
75
|
+
| **Twitter / X** | ✅ | ✅ | |
|
|
76
|
+
| **Instagram** | ✅ | ✅ | |
|
|
76
77
|
| **YouTube** | ✅ | | 🎵 音乐 |
|
|
77
|
-
| **Facebook** | ✅ | |
|
|
78
|
-
| **Threads** | ✅ | ✅ |
|
|
78
|
+
| **Facebook** | ✅ | | |
|
|
79
|
+
| **Threads** | ✅ | ✅ | |
|
|
79
80
|
| **Bilibili** | ✅ | | 📝 动态 |
|
|
80
|
-
| **抖音** | ✅ | ✅ |
|
|
81
|
-
| **TikTok** | ✅ | ✅ |
|
|
82
|
-
| **微博** | ✅ | ✅ |
|
|
83
|
-
| **小红书** | ✅ | ✅ |
|
|
84
|
-
| **贴吧** | ✅ | ✅ |
|
|
85
|
-
| **微信公众号** | | ✅ |
|
|
86
|
-
| **快手** | ✅ | |
|
|
87
|
-
| **酷安** | | ✅ |
|
|
88
|
-
| **皮皮虾** | ✅ | ✅ |
|
|
89
|
-
| **最右** | ✅ | ✅ |
|
|
90
|
-
| **小黑盒** | ✅ | ✅ |
|
|
91
|
-
| **Snapchat** | ✅ | |
|
|
81
|
+
| **抖音** | ✅ | ✅ | ☀️日常 |
|
|
82
|
+
| **TikTok** | ✅ | ✅ | |
|
|
83
|
+
| **微博** | ✅ | ✅ | |
|
|
84
|
+
| **小红书** | ✅ | ✅ | |
|
|
85
|
+
| **贴吧** | ✅ | ✅ | |
|
|
86
|
+
| **微信公众号** | | ✅ | |
|
|
87
|
+
| **快手** | ✅ | | |
|
|
88
|
+
| **酷安** | | ✅ | |
|
|
89
|
+
| **皮皮虾** | ✅ | ✅ | |
|
|
90
|
+
| **最右** | ✅ | ✅ | |
|
|
91
|
+
| **小黑盒** | ✅ | ✅ | |
|
|
92
|
+
| **Snapchat** | ✅ | | |
|
|
92
93
|
|
|
93
94
|
## 📦 安装
|
|
94
95
|
|
|
@@ -28,26 +28,26 @@
|
|
|
28
28
|
|
|
29
29
|
## 🌐 支持平台
|
|
30
30
|
|
|
31
|
-
| 平台 | 视频 | 图文 | 其他
|
|
32
|
-
|
|
33
|
-
| **Twitter / X** | ✅ | ✅ |
|
|
34
|
-
| **Instagram** | ✅ | ✅ |
|
|
31
|
+
| 平台 | 视频 | 图文 | 其他 |
|
|
32
|
+
|-----------------|:--:|:-:|------|
|
|
33
|
+
| **Twitter / X** | ✅ | ✅ | |
|
|
34
|
+
| **Instagram** | ✅ | ✅ | |
|
|
35
35
|
| **YouTube** | ✅ | | 🎵 音乐 |
|
|
36
|
-
| **Facebook** | ✅ | |
|
|
37
|
-
| **Threads** | ✅ | ✅ |
|
|
36
|
+
| **Facebook** | ✅ | | |
|
|
37
|
+
| **Threads** | ✅ | ✅ | |
|
|
38
38
|
| **Bilibili** | ✅ | | 📝 动态 |
|
|
39
|
-
| **抖音** | ✅ | ✅ |
|
|
40
|
-
| **TikTok** | ✅ | ✅ |
|
|
41
|
-
| **微博** | ✅ | ✅ |
|
|
42
|
-
| **小红书** | ✅ | ✅ |
|
|
43
|
-
| **贴吧** | ✅ | ✅ |
|
|
44
|
-
| **微信公众号** | | ✅ |
|
|
45
|
-
| **快手** | ✅ | |
|
|
46
|
-
| **酷安** | | ✅ |
|
|
47
|
-
| **皮皮虾** | ✅ | ✅ |
|
|
48
|
-
| **最右** | ✅ | ✅ |
|
|
49
|
-
| **小黑盒** | ✅ | ✅ |
|
|
50
|
-
| **Snapchat** | ✅ | |
|
|
39
|
+
| **抖音** | ✅ | ✅ | ☀️日常 |
|
|
40
|
+
| **TikTok** | ✅ | ✅ | |
|
|
41
|
+
| **微博** | ✅ | ✅ | |
|
|
42
|
+
| **小红书** | ✅ | ✅ | |
|
|
43
|
+
| **贴吧** | ✅ | ✅ | |
|
|
44
|
+
| **微信公众号** | | ✅ | |
|
|
45
|
+
| **快手** | ✅ | | |
|
|
46
|
+
| **酷安** | | ✅ | |
|
|
47
|
+
| **皮皮虾** | ✅ | ✅ | |
|
|
48
|
+
| **最右** | ✅ | ✅ | |
|
|
49
|
+
| **小黑盒** | ✅ | ✅ | |
|
|
50
|
+
| **Snapchat** | ✅ | | |
|
|
51
51
|
|
|
52
52
|
## 📦 安装
|
|
53
53
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "parsehub"
|
|
3
|
-
version = "2.0.
|
|
3
|
+
version = "2.0.41"
|
|
4
4
|
description = "轻量、异步、开箱即用的社交媒体聚合解析库"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12.0"
|
|
@@ -37,6 +37,7 @@ dependencies = [
|
|
|
37
37
|
"opencv-python-headless>=4.13.0.92",
|
|
38
38
|
"cryptography>=49.0.0",
|
|
39
39
|
"gmssl>=3.2.2",
|
|
40
|
+
"SignerPy>=0.12.0",
|
|
40
41
|
]
|
|
41
42
|
|
|
42
43
|
[project.scripts]
|
|
@@ -4,7 +4,7 @@ from pathlib import Path
|
|
|
4
4
|
from typing import Self, Union
|
|
5
5
|
|
|
6
6
|
from ... import ProgressCallback
|
|
7
|
-
from ...provider_api.douyin import DouyinWebCrawler
|
|
7
|
+
from ...provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice, DouyinWebCrawler
|
|
8
8
|
from ...types import (
|
|
9
9
|
DownloadResult,
|
|
10
10
|
ImageParseResult,
|
|
@@ -22,7 +22,7 @@ from ..base.base import BaseParser
|
|
|
22
22
|
|
|
23
23
|
class DouyinParser(BaseParser):
|
|
24
24
|
__platform__ = Platform.DOUYIN
|
|
25
|
-
__supported_type__ = ["视频", "图文"]
|
|
25
|
+
__supported_type__ = ["视频", "图文", "日常"]
|
|
26
26
|
__match__ = r"^(http(s)?://)?.+douyin.com/(?!share/user|qishui).+"
|
|
27
27
|
__redirect_keywords__ = ["v.douyin", "iesdouyin"]
|
|
28
28
|
__reserved_parameters__ = ["modal_id"]
|
|
@@ -36,14 +36,30 @@ class DouyinParser(BaseParser):
|
|
|
36
36
|
case DouyinMediaType.IMAGE:
|
|
37
37
|
return self._build_image_result(result)
|
|
38
38
|
|
|
39
|
-
async def _fetch_api_result(self,
|
|
39
|
+
async def _fetch_api_result(self, raw_url: str) -> "DouyinApiResult":
|
|
40
40
|
"""获取并解析抖音 API 结果"""
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
web_cookie = self.cookie.get_value() or {}
|
|
42
|
+
|
|
43
|
+
web_error: ParseError | None = None
|
|
44
|
+
if web_cookie:
|
|
45
|
+
try:
|
|
46
|
+
web_crawler = DouyinWebCrawler(proxy=self.proxy, cookie=web_cookie)
|
|
47
|
+
response = await web_crawler.parse(raw_url)
|
|
48
|
+
return DouyinApiResult.parse(response)
|
|
49
|
+
except ParseError as e:
|
|
50
|
+
web_error = e
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
mobile_device = DouyinMobileDevice.resolve()
|
|
54
|
+
app_crawler = DouyinMobileCrawler(proxy=self.proxy, device=mobile_device)
|
|
55
|
+
response = await app_crawler.parse(raw_url)
|
|
56
|
+
return DouyinApiResult.parse(response)
|
|
57
|
+
except ParseError as e:
|
|
58
|
+
mobile_error = e
|
|
59
|
+
|
|
60
|
+
if web_error:
|
|
61
|
+
raise ParseError(f"{web_error};移动端解析也失败: {mobile_error}") from mobile_error
|
|
62
|
+
raise mobile_error
|
|
47
63
|
|
|
48
64
|
@staticmethod
|
|
49
65
|
def _build_video_result(result: "DouyinApiResult") -> VideoParseResult:
|
|
@@ -106,9 +122,14 @@ def parse_video_info(video_data: dict) -> dict:
|
|
|
106
122
|
if not bit_rates:
|
|
107
123
|
raise ParseError("抖音解析失败: 未获取到视频下载地址")
|
|
108
124
|
|
|
109
|
-
#
|
|
125
|
+
# 按分辨率、文件大小、码率降序排列,选择最高质量。
|
|
126
|
+
# Story/日常的移动端 default 播放地址通常没有 br 参数,但 data_size 最大。
|
|
110
127
|
bit_rates.sort(
|
|
111
|
-
key=lambda x:
|
|
128
|
+
key=lambda x: (
|
|
129
|
+
x.get("play_addr", {}).get("width", 0) * x.get("play_addr", {}).get("height", 0),
|
|
130
|
+
x.get("play_addr", {}).get("data_size", 0),
|
|
131
|
+
x.get("bit_rate", 0),
|
|
132
|
+
),
|
|
112
133
|
reverse=True,
|
|
113
134
|
)
|
|
114
135
|
best_quality = bit_rates[0]
|
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
# mypy: disable-error-code=no-untyped-def
|
|
2
|
+
# mypy: disable-error-code=no-any-return
|
|
2
3
|
import asyncio
|
|
3
4
|
import base64
|
|
5
|
+
import binascii
|
|
4
6
|
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
5
9
|
import random
|
|
6
10
|
import re
|
|
7
11
|
import time
|
|
12
|
+
import uuid
|
|
13
|
+
from dataclasses import dataclass
|
|
8
14
|
from random import choice, randint
|
|
9
|
-
from typing import Any, cast
|
|
15
|
+
from typing import Any, ClassVar, cast
|
|
10
16
|
from urllib.parse import quote, urlencode
|
|
11
17
|
|
|
12
18
|
import httpx
|
|
13
19
|
from gmssl import func, sm3
|
|
20
|
+
from SignerPy import get, sign, trace_id
|
|
14
21
|
|
|
15
22
|
from ..errors import ParseError
|
|
16
23
|
|
|
@@ -20,6 +27,35 @@ DEFAULT_USER_AGENT = (
|
|
|
20
27
|
)
|
|
21
28
|
|
|
22
29
|
POST_DETAIL = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
|
|
30
|
+
MOBILE_BROWSER_USER_AGENT = (
|
|
31
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
|
|
32
|
+
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1"
|
|
33
|
+
)
|
|
34
|
+
MOBILE_USER_AGENT = (
|
|
35
|
+
"com.ss.android.ugc.aweme/390500 (Linux; U; Android 13; zh_CN; Pixel 6; "
|
|
36
|
+
"Build/TQ3A.230805.001; Cronet/TTNetVersion:6b6f6e6e 2024-04-10 "
|
|
37
|
+
"QuicVersion:47946d2a 2024-03-28)"
|
|
38
|
+
)
|
|
39
|
+
PLAY_USER_AGENT = (
|
|
40
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
41
|
+
)
|
|
42
|
+
MOBILE_DETAIL_HOSTS = (
|
|
43
|
+
"api.amemv.com",
|
|
44
|
+
"api3-core-c.amemv.com",
|
|
45
|
+
"aweme.snssdk.com",
|
|
46
|
+
"api5-normal-lf.amemv.com",
|
|
47
|
+
"api3-normal-c.amemv.com",
|
|
48
|
+
)
|
|
49
|
+
MOBILE_REGISTER_HOSTS = (
|
|
50
|
+
"log.snssdk.com",
|
|
51
|
+
"api.amemv.com",
|
|
52
|
+
)
|
|
53
|
+
MOBILE_DEVICE_POOL_SIZE = 3
|
|
54
|
+
MOBILE_SIGN_PROFILES = (
|
|
55
|
+
{"license_id": 1611921764, "version": 8404},
|
|
56
|
+
{"license_id": 1611921764, "version": 4404},
|
|
57
|
+
)
|
|
58
|
+
MOBILE_PLAY_RATIOS = ("default", "1080p", "720p", "540p", "480p")
|
|
23
59
|
|
|
24
60
|
|
|
25
61
|
class XBogus:
|
|
@@ -718,21 +754,18 @@ class DouyinWebCrawler:
|
|
|
718
754
|
"Referer": "https://www.douyin.com/",
|
|
719
755
|
}
|
|
720
756
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
if match:
|
|
734
|
-
return match.group(1)
|
|
735
|
-
raise ValueError("未在响应的地址中找到 aweme_id")
|
|
757
|
+
@staticmethod
|
|
758
|
+
async def get_aweme_id(raw_url: str) -> str:
|
|
759
|
+
for pattern in [
|
|
760
|
+
re.compile(r"video/([^/?]*)"),
|
|
761
|
+
re.compile(r"[?&]vid=(\d+)"),
|
|
762
|
+
re.compile(r"note/([^/?]*)"),
|
|
763
|
+
re.compile(r"modal_id=([0-9]+)"),
|
|
764
|
+
]:
|
|
765
|
+
match = pattern.search(raw_url)
|
|
766
|
+
if match:
|
|
767
|
+
return match.group(1)
|
|
768
|
+
raise ValueError("未在响应的地址中找到 aweme_id")
|
|
736
769
|
|
|
737
770
|
async def fetch_one_video(self, aweme_id: str) -> dict:
|
|
738
771
|
async with httpx.AsyncClient(
|
|
@@ -778,6 +811,404 @@ class DouyinWebCrawler:
|
|
|
778
811
|
raise ParseError("获取抖音作品失败, 请检查 cookie") from e
|
|
779
812
|
raise ParseError("获取抖音作品失败, 请检查 cookie")
|
|
780
813
|
|
|
781
|
-
async def parse(self,
|
|
782
|
-
aweme_id = await self.get_aweme_id(
|
|
814
|
+
async def parse(self, raw_url: str) -> dict:
|
|
815
|
+
aweme_id = await self.get_aweme_id(raw_url)
|
|
816
|
+
return await self.fetch_one_video(aweme_id)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
@dataclass(frozen=True)
|
|
820
|
+
class DouyinMobileDevice:
|
|
821
|
+
"""抖音移动端设备注册参数。
|
|
822
|
+
|
|
823
|
+
Story/日常作品在网页分享页会返回 story_25_filter,无法拿到 aweme_detail。
|
|
824
|
+
移动端详情接口需要先向 device_register 注册 device_id/iid;
|
|
825
|
+
随机生成的 device_id/iid 通常会返回 200 空 body。
|
|
826
|
+
"""
|
|
827
|
+
|
|
828
|
+
device_id: str
|
|
829
|
+
iid: str
|
|
830
|
+
cdid: str | None = None
|
|
831
|
+
openudid: str | None = None
|
|
832
|
+
|
|
833
|
+
@classmethod
|
|
834
|
+
def from_env(cls) -> "DouyinMobileDevice | None":
|
|
835
|
+
device_id = os.getenv("PARSEHUB_DOUYIN_DEVICE_ID") or os.getenv("DOUYIN_DEVICE_ID")
|
|
836
|
+
iid = os.getenv("PARSEHUB_DOUYIN_IID") or os.getenv("PARSEHUB_DOUYIN_INSTALL_ID") or os.getenv("DOUYIN_IID")
|
|
837
|
+
if not device_id or not iid:
|
|
838
|
+
return None
|
|
839
|
+
cdid = os.getenv("PARSEHUB_DOUYIN_CDID") or os.getenv("DOUYIN_CDID")
|
|
840
|
+
openudid = os.getenv("PARSEHUB_DOUYIN_OPENUDID") or os.getenv("DOUYIN_OPENUDID")
|
|
841
|
+
return cls(
|
|
842
|
+
device_id=device_id.strip(),
|
|
843
|
+
iid=iid.strip(),
|
|
844
|
+
cdid=cdid.strip() if cdid else None,
|
|
845
|
+
openudid=openudid.strip() if openudid else None,
|
|
846
|
+
)
|
|
847
|
+
|
|
848
|
+
@classmethod
|
|
849
|
+
def resolve(cls) -> "DouyinMobileDevice | None":
|
|
850
|
+
return cls.from_env()
|
|
851
|
+
|
|
852
|
+
@classmethod
|
|
853
|
+
def from_register_response(
|
|
854
|
+
cls,
|
|
855
|
+
payload: dict[str, Any],
|
|
856
|
+
*,
|
|
857
|
+
cdid: str | None = None,
|
|
858
|
+
openudid: str | None = None,
|
|
859
|
+
) -> "DouyinMobileDevice | None":
|
|
860
|
+
device_id = str(payload.get("device_id_str") or payload.get("device_id") or "")
|
|
861
|
+
iid = str(payload.get("install_id_str") or payload.get("install_id") or payload.get("iid") or "")
|
|
862
|
+
if not device_id or not iid or device_id == "0" or iid == "0":
|
|
863
|
+
return None
|
|
864
|
+
return cls(device_id=device_id, iid=iid, cdid=cdid, openudid=openudid)
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
class DouyinMobileCrawler:
|
|
868
|
+
"""Signed mobile API crawler for Douyin Story/日常 videos."""
|
|
869
|
+
|
|
870
|
+
_device_pool: ClassVar[list[DouyinMobileDevice]] = []
|
|
871
|
+
_device_pool_index: ClassVar[int] = 0
|
|
872
|
+
|
|
873
|
+
def __init__(self, device: DouyinMobileDevice | None = None, proxy: str | None = None):
|
|
874
|
+
self.device = device
|
|
875
|
+
self._fixed_device = device is not None
|
|
876
|
+
self.proxy = proxy
|
|
877
|
+
|
|
878
|
+
@staticmethod
|
|
879
|
+
async def get_aweme_id(raw_url: str) -> str:
|
|
880
|
+
for pattern in [
|
|
881
|
+
re.compile(r"/(?:share/)?video/(\d+)"),
|
|
882
|
+
re.compile(r"[?&]modal_id=(\d+)"),
|
|
883
|
+
re.compile(r"note/([^/?]*)"),
|
|
884
|
+
re.compile(r"[?&]vid=(\d+)"),
|
|
885
|
+
]:
|
|
886
|
+
match = pattern.search(raw_url)
|
|
887
|
+
if match:
|
|
888
|
+
return match.group(1)
|
|
889
|
+
raise ParseError("未在响应地址中找到 aweme_id")
|
|
890
|
+
|
|
891
|
+
def _mobile_query(self, aweme_id: str) -> dict:
|
|
892
|
+
if self.device is None:
|
|
893
|
+
raise ParseError("抖音移动端设备未注册")
|
|
894
|
+
|
|
895
|
+
params = get(
|
|
896
|
+
{
|
|
897
|
+
"aweme_id": aweme_id,
|
|
898
|
+
"aid": "1128",
|
|
899
|
+
"app_name": "aweme",
|
|
900
|
+
"version_code": "390500",
|
|
901
|
+
"version_name": "39.5.0",
|
|
902
|
+
"device_platform": "android",
|
|
903
|
+
"os": "android",
|
|
904
|
+
"os_version": "13",
|
|
905
|
+
"ssmix": "a",
|
|
906
|
+
"language": "zh",
|
|
907
|
+
"channel": "wandoujia_aweme",
|
|
908
|
+
"device_type": "Pixel 6",
|
|
909
|
+
"device_brand": "google",
|
|
910
|
+
"resolution": "1080*2400",
|
|
911
|
+
"dpi": "420",
|
|
912
|
+
"host_abi": "arm64-v8a",
|
|
913
|
+
"manifest_version_code": "390500",
|
|
914
|
+
"update_version_code": "390500",
|
|
915
|
+
"ac": "wifi",
|
|
916
|
+
"is_guest_mode": "0",
|
|
917
|
+
"minor_status": "0",
|
|
918
|
+
"app_type": "normal",
|
|
919
|
+
}
|
|
920
|
+
)
|
|
921
|
+
params["device_id"] = self.device.device_id
|
|
922
|
+
params["iid"] = self.device.iid
|
|
923
|
+
if self.device.cdid:
|
|
924
|
+
params["cdid"] = self.device.cdid
|
|
925
|
+
if self.device.openudid:
|
|
926
|
+
params["openudid"] = self.device.openudid
|
|
927
|
+
return params
|
|
928
|
+
|
|
929
|
+
def _signed_headers(self, params: dict, profile: dict) -> dict[str, str]:
|
|
930
|
+
query = urlencode(params)
|
|
931
|
+
signed = sign(
|
|
932
|
+
params=query,
|
|
933
|
+
aid=1128,
|
|
934
|
+
license_id=profile["license_id"],
|
|
935
|
+
version=profile["version"],
|
|
936
|
+
platform=0,
|
|
937
|
+
sdk_version_str="v05.01.02-alpha.7-ov-android",
|
|
938
|
+
sdk_version=83952160,
|
|
939
|
+
)
|
|
940
|
+
return {
|
|
941
|
+
**signed,
|
|
942
|
+
"User-Agent": MOBILE_USER_AGENT,
|
|
943
|
+
"x-tt-trace-id": trace_id(params["device_id"]),
|
|
944
|
+
"sdk-version": "2",
|
|
945
|
+
"passport-sdk-version": "203226",
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
@staticmethod
|
|
949
|
+
def _new_openudid() -> str:
|
|
950
|
+
return binascii.hexlify(os.urandom(8)).decode()
|
|
951
|
+
|
|
952
|
+
@staticmethod
|
|
953
|
+
def _device_register_query() -> dict[str, str | int]:
|
|
954
|
+
params = get(
|
|
955
|
+
{
|
|
956
|
+
"aid": "1128",
|
|
957
|
+
"app_name": "aweme",
|
|
958
|
+
"version_code": "390500",
|
|
959
|
+
"version_name": "39.5.0",
|
|
960
|
+
"device_platform": "android",
|
|
961
|
+
"os": "android",
|
|
962
|
+
"os_version": "13",
|
|
963
|
+
"ssmix": "a",
|
|
964
|
+
"language": "zh",
|
|
965
|
+
"channel": "wandoujia_aweme",
|
|
966
|
+
"device_type": "Pixel 6",
|
|
967
|
+
"device_brand": "google",
|
|
968
|
+
"resolution": "1080*2400",
|
|
969
|
+
"dpi": "420",
|
|
970
|
+
"host_abi": "arm64-v8a",
|
|
971
|
+
"manifest_version_code": "390500",
|
|
972
|
+
"update_version_code": "390500",
|
|
973
|
+
"ac": "wifi",
|
|
974
|
+
"app_type": "normal",
|
|
975
|
+
"cpu_support64": "true",
|
|
976
|
+
}
|
|
977
|
+
)
|
|
978
|
+
return params
|
|
979
|
+
|
|
980
|
+
@staticmethod
|
|
981
|
+
def _device_register_payload(params: dict[str, str | int]) -> dict:
|
|
982
|
+
return {
|
|
983
|
+
"magic_tag": "ss_app_log",
|
|
984
|
+
"header": {
|
|
985
|
+
"display_name": "抖音",
|
|
986
|
+
"aid": 1128,
|
|
987
|
+
"channel": "wandoujia_aweme",
|
|
988
|
+
"package": "com.ss.android.ugc.aweme",
|
|
989
|
+
"app_version": "39.5.0",
|
|
990
|
+
"version_code": 390500,
|
|
991
|
+
"manifest_version_code": 390500,
|
|
992
|
+
"update_version_code": 390500,
|
|
993
|
+
"sdk_version": "3.9.5",
|
|
994
|
+
"sdk_target_version": 29,
|
|
995
|
+
"os": "Android",
|
|
996
|
+
"os_version": "13",
|
|
997
|
+
"os_api": 33,
|
|
998
|
+
"device_model": "Pixel 6",
|
|
999
|
+
"device_brand": "google",
|
|
1000
|
+
"device_manufacturer": "Google",
|
|
1001
|
+
"cpu_abi": "arm64-v8a",
|
|
1002
|
+
"release_build": "TQ3A.230805.001",
|
|
1003
|
+
"density_dpi": 420,
|
|
1004
|
+
"display_density": "xhdpi",
|
|
1005
|
+
"resolution": "1080x2400",
|
|
1006
|
+
"language": "zh",
|
|
1007
|
+
"timezone": 8,
|
|
1008
|
+
"region": "CN",
|
|
1009
|
+
"tz_name": "Asia/Shanghai",
|
|
1010
|
+
"cdid": params["cdid"],
|
|
1011
|
+
"openudid": params["openudid"],
|
|
1012
|
+
"clientudid": str(uuid.uuid4()),
|
|
1013
|
+
"google_aid": "",
|
|
1014
|
+
"req_id": str(uuid.uuid4()),
|
|
1015
|
+
},
|
|
1016
|
+
"_gen_time": int(time.time()),
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
async def _request_registered_device(self, client: httpx.AsyncClient) -> DouyinMobileDevice:
|
|
1020
|
+
last_error = "unknown"
|
|
1021
|
+
for host in MOBILE_REGISTER_HOSTS:
|
|
1022
|
+
params = self._device_register_query()
|
|
1023
|
+
query = urlencode(params)
|
|
1024
|
+
url = f"https://{host}/service/2/device_register/?{query}"
|
|
1025
|
+
headers = {
|
|
1026
|
+
**self._signed_headers(params, MOBILE_SIGN_PROFILES[0]),
|
|
1027
|
+
"User-Agent": MOBILE_USER_AGENT,
|
|
1028
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
1029
|
+
"sdk-version": "2",
|
|
1030
|
+
}
|
|
1031
|
+
body = json.dumps(self._device_register_payload(params), ensure_ascii=False, separators=(",", ":"))
|
|
1032
|
+
try:
|
|
1033
|
+
response = await client.post(url, headers=headers, content=body.encode(), timeout=20)
|
|
1034
|
+
payload = response.json()
|
|
1035
|
+
except Exception as e:
|
|
1036
|
+
last_error = str(e)
|
|
1037
|
+
continue
|
|
1038
|
+
|
|
1039
|
+
device = DouyinMobileDevice.from_register_response(
|
|
1040
|
+
payload,
|
|
1041
|
+
cdid=str(params["cdid"]),
|
|
1042
|
+
openudid=str(params["openudid"]),
|
|
1043
|
+
)
|
|
1044
|
+
if device:
|
|
1045
|
+
return device
|
|
1046
|
+
last_error = f"{host} returned invalid device ids: {payload}"
|
|
1047
|
+
raise ParseError(f"注册抖音移动端设备失败: {last_error}")
|
|
1048
|
+
|
|
1049
|
+
async def register_device(self, client: httpx.AsyncClient | None = None) -> DouyinMobileDevice:
|
|
1050
|
+
close_client = client is None
|
|
1051
|
+
client = client or httpx.AsyncClient(proxy=self.proxy, timeout=20, follow_redirects=True)
|
|
1052
|
+
try:
|
|
1053
|
+
device = await self._request_registered_device(client)
|
|
1054
|
+
self.device = device
|
|
1055
|
+
self._fixed_device = True
|
|
1056
|
+
return self.device
|
|
1057
|
+
finally:
|
|
1058
|
+
if close_client:
|
|
1059
|
+
await client.aclose()
|
|
1060
|
+
|
|
1061
|
+
async def register_device_pool(
|
|
1062
|
+
self,
|
|
1063
|
+
client: httpx.AsyncClient | None = None,
|
|
1064
|
+
*,
|
|
1065
|
+
count: int = MOBILE_DEVICE_POOL_SIZE,
|
|
1066
|
+
) -> list[DouyinMobileDevice]:
|
|
1067
|
+
close_client = client is None
|
|
1068
|
+
client = client or httpx.AsyncClient(proxy=self.proxy, timeout=20, follow_redirects=True)
|
|
1069
|
+
devices: list[DouyinMobileDevice] = []
|
|
1070
|
+
seen: set[tuple[str, str]] = set()
|
|
1071
|
+
last_error = "unknown"
|
|
1072
|
+
try:
|
|
1073
|
+
attempts = 0
|
|
1074
|
+
while len(devices) < count and attempts < count * 10:
|
|
1075
|
+
attempts += 1
|
|
1076
|
+
try:
|
|
1077
|
+
device = await self._request_registered_device(client)
|
|
1078
|
+
except ParseError as e:
|
|
1079
|
+
last_error = str(e)
|
|
1080
|
+
await asyncio.sleep(0.2)
|
|
1081
|
+
continue
|
|
1082
|
+
key = (device.device_id, device.iid)
|
|
1083
|
+
if key in seen:
|
|
1084
|
+
last_error = f"duplicate device ids: {device.device_id}/{device.iid}"
|
|
1085
|
+
await asyncio.sleep(0.2)
|
|
1086
|
+
continue
|
|
1087
|
+
seen.add(key)
|
|
1088
|
+
devices.append(device)
|
|
1089
|
+
finally:
|
|
1090
|
+
if close_client:
|
|
1091
|
+
await client.aclose()
|
|
1092
|
+
if len(devices) < count:
|
|
1093
|
+
raise ParseError(f"注册抖音移动端设备池失败: 仅获取到 {len(devices)}/{count} 组设备({last_error})")
|
|
1094
|
+
return devices
|
|
1095
|
+
|
|
1096
|
+
@classmethod
|
|
1097
|
+
def _next_pooled_device(cls) -> DouyinMobileDevice:
|
|
1098
|
+
if not cls._device_pool:
|
|
1099
|
+
raise ParseError("抖音移动端设备池未初始化")
|
|
1100
|
+
device = cls._device_pool[cls._device_pool_index % len(cls._device_pool)]
|
|
1101
|
+
cls._device_pool_index = (cls._device_pool_index + 1) % len(cls._device_pool)
|
|
1102
|
+
return device
|
|
1103
|
+
|
|
1104
|
+
async def _ensure_device_pool(self, client: httpx.AsyncClient) -> None:
|
|
1105
|
+
if self.__class__._device_pool:
|
|
1106
|
+
return
|
|
1107
|
+
self.__class__._device_pool = await self.register_device_pool(client)
|
|
1108
|
+
self.__class__._device_pool_index = 0
|
|
1109
|
+
|
|
1110
|
+
async def _select_device(self, client: httpx.AsyncClient) -> DouyinMobileDevice:
|
|
1111
|
+
if self._fixed_device:
|
|
1112
|
+
if self.device is None:
|
|
1113
|
+
raise ParseError("抖音移动端设备未配置")
|
|
1114
|
+
return self.device
|
|
1115
|
+
|
|
1116
|
+
await self._ensure_device_pool(client)
|
|
1117
|
+
self.device = self.__class__._next_pooled_device()
|
|
1118
|
+
return self.device
|
|
1119
|
+
|
|
1120
|
+
async def fetch_one_video(self, aweme_id: str) -> dict:
|
|
1121
|
+
last_error = "unknown"
|
|
1122
|
+
async with httpx.AsyncClient(proxy=self.proxy, timeout=20, follow_redirects=True) as client:
|
|
1123
|
+
for _ in range(8):
|
|
1124
|
+
await self._select_device(client)
|
|
1125
|
+
params = self._mobile_query(aweme_id)
|
|
1126
|
+
query = urlencode(params)
|
|
1127
|
+
for profile in MOBILE_SIGN_PROFILES:
|
|
1128
|
+
headers = self._signed_headers(params, profile)
|
|
1129
|
+
for host in MOBILE_DETAIL_HOSTS:
|
|
1130
|
+
url = f"https://{host}/aweme/v1/aweme/detail/?{query}"
|
|
1131
|
+
try:
|
|
1132
|
+
response = await client.get(url, headers=headers)
|
|
1133
|
+
content = response.content
|
|
1134
|
+
except Exception as e:
|
|
1135
|
+
last_error = str(e)
|
|
1136
|
+
continue
|
|
1137
|
+
if not content:
|
|
1138
|
+
last_error = f"{host} returned empty body"
|
|
1139
|
+
continue
|
|
1140
|
+
try:
|
|
1141
|
+
payload = response.json()
|
|
1142
|
+
except Exception:
|
|
1143
|
+
last_error = f"{host} returned non-json body"
|
|
1144
|
+
continue
|
|
1145
|
+
if payload.get("aweme_detail"):
|
|
1146
|
+
await self._attach_story_default_play(payload["aweme_detail"])
|
|
1147
|
+
return cast(dict[str, Any], payload)
|
|
1148
|
+
last_error = f"{host} missing aweme_detail: {payload.get('status_msg') or payload}"
|
|
1149
|
+
await asyncio.sleep(0.15)
|
|
1150
|
+
raise ParseError(f"获取抖音 Story/日常失败: {last_error}")
|
|
1151
|
+
|
|
1152
|
+
async def parse(self, raw_url: str) -> dict:
|
|
1153
|
+
aweme_id = await self.get_aweme_id(raw_url)
|
|
783
1154
|
return await self.fetch_one_video(aweme_id)
|
|
1155
|
+
|
|
1156
|
+
async def _attach_story_default_play(self, detail: dict) -> None:
|
|
1157
|
+
if not (detail.get("is_story") in (1, True, "1") or detail.get("is_24_story") in (1, True, "1")):
|
|
1158
|
+
return
|
|
1159
|
+
video = detail.get("video") or {}
|
|
1160
|
+
video_uri = self._pick_video_uri(video)
|
|
1161
|
+
if not video_uri:
|
|
1162
|
+
return
|
|
1163
|
+
best = await self._resolve_best_play_url(video_uri)
|
|
1164
|
+
if not best:
|
|
1165
|
+
return
|
|
1166
|
+
|
|
1167
|
+
play_addr = {
|
|
1168
|
+
"uri": video_uri,
|
|
1169
|
+
"url_list": [best["direct_url"]],
|
|
1170
|
+
"data_size": best["content_length"],
|
|
1171
|
+
"width": video.get("width") or 0,
|
|
1172
|
+
"height": video.get("height") or 0,
|
|
1173
|
+
}
|
|
1174
|
+
synthetic_best = {
|
|
1175
|
+
"bit_rate": best["bitrate_kbps"] or 0,
|
|
1176
|
+
"duration": video.get("duration") or 0,
|
|
1177
|
+
"play_addr": play_addr,
|
|
1178
|
+
}
|
|
1179
|
+
video["bit_rate"] = [synthetic_best, *(video.get("bit_rate") or [])]
|
|
1180
|
+
|
|
1181
|
+
@staticmethod
|
|
1182
|
+
def _pick_video_uri(video: dict) -> str:
|
|
1183
|
+
if uri := (video.get("play_addr", {})).get("uri"):
|
|
1184
|
+
return str(uri)
|
|
1185
|
+
best_uri = ""
|
|
1186
|
+
best_bitrate = -1
|
|
1187
|
+
for item in video.get("bit_rate", []):
|
|
1188
|
+
candidate = (item.get("play_addr", {})).get("uri")
|
|
1189
|
+
bitrate = int(item.get("bit_rate", 0))
|
|
1190
|
+
if candidate and bitrate >= best_bitrate:
|
|
1191
|
+
best_uri = candidate
|
|
1192
|
+
best_bitrate = bitrate
|
|
1193
|
+
return best_uri
|
|
1194
|
+
|
|
1195
|
+
async def _resolve_best_play_url(self, video_uri: str) -> dict | None:
|
|
1196
|
+
headers = {"User-Agent": PLAY_USER_AGENT, "Referer": "https://www.douyin.com/"}
|
|
1197
|
+
best: dict | None = None
|
|
1198
|
+
async with httpx.AsyncClient(proxy=self.proxy, timeout=20, follow_redirects=True) as client:
|
|
1199
|
+
for ratio in MOBILE_PLAY_RATIOS:
|
|
1200
|
+
api = f"https://aweme.snssdk.com/aweme/v1/play/?video_id={video_uri}&ratio={ratio}&line=0"
|
|
1201
|
+
try:
|
|
1202
|
+
response = await client.head(api, headers=headers)
|
|
1203
|
+
except Exception:
|
|
1204
|
+
continue
|
|
1205
|
+
content_length = int(response.headers.get("content-length", 0))
|
|
1206
|
+
candidate = {
|
|
1207
|
+
"ratio": ratio,
|
|
1208
|
+
"direct_url": str(response.url),
|
|
1209
|
+
"content_length": content_length,
|
|
1210
|
+
"bitrate_kbps": 0,
|
|
1211
|
+
}
|
|
1212
|
+
if best is None or candidate["content_length"] > best["content_length"]:
|
|
1213
|
+
best = candidate
|
|
1214
|
+
return best
|
|
@@ -49,8 +49,10 @@ class SecretCookie:
|
|
|
49
49
|
|
|
50
50
|
@staticmethod
|
|
51
51
|
def normalize_cookie(v: str | dict[str, Any] | None) -> dict[str, SecretStr] | None:
|
|
52
|
-
if v is None
|
|
52
|
+
if v is None:
|
|
53
53
|
return v
|
|
54
|
+
if isinstance(v, dict):
|
|
55
|
+
return {str(k).strip(): SecretStr("" if v is None else str(v).strip()) for k, v in v.items()}
|
|
54
56
|
if isinstance(v, str):
|
|
55
57
|
s = v.strip()
|
|
56
58
|
if not s:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.41
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -34,6 +34,7 @@ Requires-Dist: python-slugify[unidecode]>=8.0.4
|
|
|
34
34
|
Requires-Dist: opencv-python-headless>=4.13.0.92
|
|
35
35
|
Requires-Dist: cryptography>=49.0.0
|
|
36
36
|
Requires-Dist: gmssl>=3.2.2
|
|
37
|
+
Requires-Dist: SignerPy>=0.12.0
|
|
37
38
|
Provides-Extra: cli
|
|
38
39
|
Requires-Dist: argcomplete>=3.6.3; extra == "cli"
|
|
39
40
|
Requires-Dist: platformdirs>=4.5.1; extra == "cli"
|
|
@@ -69,26 +70,26 @@ Dynamic: license-file
|
|
|
69
70
|
|
|
70
71
|
## 🌐 支持平台
|
|
71
72
|
|
|
72
|
-
| 平台 | 视频 | 图文 | 其他
|
|
73
|
-
|
|
74
|
-
| **Twitter / X** | ✅ | ✅ |
|
|
75
|
-
| **Instagram** | ✅ | ✅ |
|
|
73
|
+
| 平台 | 视频 | 图文 | 其他 |
|
|
74
|
+
|-----------------|:--:|:-:|------|
|
|
75
|
+
| **Twitter / X** | ✅ | ✅ | |
|
|
76
|
+
| **Instagram** | ✅ | ✅ | |
|
|
76
77
|
| **YouTube** | ✅ | | 🎵 音乐 |
|
|
77
|
-
| **Facebook** | ✅ | |
|
|
78
|
-
| **Threads** | ✅ | ✅ |
|
|
78
|
+
| **Facebook** | ✅ | | |
|
|
79
|
+
| **Threads** | ✅ | ✅ | |
|
|
79
80
|
| **Bilibili** | ✅ | | 📝 动态 |
|
|
80
|
-
| **抖音** | ✅ | ✅ |
|
|
81
|
-
| **TikTok** | ✅ | ✅ |
|
|
82
|
-
| **微博** | ✅ | ✅ |
|
|
83
|
-
| **小红书** | ✅ | ✅ |
|
|
84
|
-
| **贴吧** | ✅ | ✅ |
|
|
85
|
-
| **微信公众号** | | ✅ |
|
|
86
|
-
| **快手** | ✅ | |
|
|
87
|
-
| **酷安** | | ✅ |
|
|
88
|
-
| **皮皮虾** | ✅ | ✅ |
|
|
89
|
-
| **最右** | ✅ | ✅ |
|
|
90
|
-
| **小黑盒** | ✅ | ✅ |
|
|
91
|
-
| **Snapchat** | ✅ | |
|
|
81
|
+
| **抖音** | ✅ | ✅ | ☀️日常 |
|
|
82
|
+
| **TikTok** | ✅ | ✅ | |
|
|
83
|
+
| **微博** | ✅ | ✅ | |
|
|
84
|
+
| **小红书** | ✅ | ✅ | |
|
|
85
|
+
| **贴吧** | ✅ | ✅ | |
|
|
86
|
+
| **微信公众号** | | ✅ | |
|
|
87
|
+
| **快手** | ✅ | | |
|
|
88
|
+
| **酷安** | | ✅ | |
|
|
89
|
+
| **皮皮虾** | ✅ | ✅ | |
|
|
90
|
+
| **最右** | ✅ | ✅ | |
|
|
91
|
+
| **小黑盒** | ✅ | ✅ | |
|
|
92
|
+
| **Snapchat** | ✅ | | |
|
|
92
93
|
|
|
93
94
|
## 📦 安装
|
|
94
95
|
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import unittest
|
|
2
|
+
from unittest.mock import patch
|
|
2
3
|
from urllib.parse import parse_qs, urlparse
|
|
3
4
|
|
|
4
5
|
from parsehub import ParseHub
|
|
5
6
|
from parsehub.errors import ParseError, UnknownPlatform
|
|
6
7
|
from parsehub.parsers.base import BaseParser
|
|
8
|
+
from parsehub.parsers.parser.douyin import parse_video_info
|
|
9
|
+
from parsehub.provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice
|
|
7
10
|
from parsehub.types import ImageParseResult, ImageRef, Platform, VideoParseResult, VideoRef
|
|
8
|
-
from parsehub.utils.helpers import match_url, run_sync
|
|
11
|
+
from parsehub.utils.helpers import SecretCookie, match_url, run_sync
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
class DummyParser(BaseParser):
|
|
@@ -68,6 +71,11 @@ class TestCoreUtilities(unittest.TestCase):
|
|
|
68
71
|
|
|
69
72
|
self.assertEqual(run_sync(get_value()), "ok")
|
|
70
73
|
|
|
74
|
+
def test_secret_cookie_accepts_dict_input(self):
|
|
75
|
+
cookie = SecretCookie({"device_id": "123", "iid": "456"})
|
|
76
|
+
|
|
77
|
+
self.assertEqual(cookie.get_value(), {"device_id": "123", "iid": "456"})
|
|
78
|
+
|
|
71
79
|
|
|
72
80
|
class TestBaseParserUrlCleaning(unittest.IsolatedAsyncioTestCase):
|
|
73
81
|
async def test_get_raw_url_keeps_reserved_and_parse_only_parameters(self):
|
|
@@ -201,6 +209,94 @@ class TestParseResultToDict(unittest.TestCase):
|
|
|
201
209
|
)
|
|
202
210
|
|
|
203
211
|
|
|
212
|
+
class TestDouyinStorySupport(unittest.TestCase):
|
|
213
|
+
def test_mobile_device_can_be_loaded_from_env(self):
|
|
214
|
+
with patch.dict(
|
|
215
|
+
"os.environ",
|
|
216
|
+
{
|
|
217
|
+
"PARSEHUB_DOUYIN_DEVICE_ID": "1325343490970016",
|
|
218
|
+
"PARSEHUB_DOUYIN_IID": "1325343490974112",
|
|
219
|
+
},
|
|
220
|
+
clear=True,
|
|
221
|
+
):
|
|
222
|
+
device = DouyinMobileDevice.from_env()
|
|
223
|
+
|
|
224
|
+
self.assertIsNotNone(device)
|
|
225
|
+
assert device is not None
|
|
226
|
+
self.assertEqual(device.device_id, "1325343490970016")
|
|
227
|
+
self.assertEqual(device.iid, "1325343490974112")
|
|
228
|
+
|
|
229
|
+
def test_mobile_device_can_be_loaded_from_register_response(self):
|
|
230
|
+
device = DouyinMobileDevice.from_register_response(
|
|
231
|
+
{
|
|
232
|
+
"device_id": 1219790537803243,
|
|
233
|
+
"install_id": 1219790537807339,
|
|
234
|
+
"device_id_str": "1219790537803243",
|
|
235
|
+
"install_id_str": "1219790537807339",
|
|
236
|
+
},
|
|
237
|
+
cdid="demo-cdid",
|
|
238
|
+
openudid="demo-openudid",
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
self.assertIsNotNone(device)
|
|
242
|
+
assert device is not None
|
|
243
|
+
self.assertEqual(device.device_id, "1219790537803243")
|
|
244
|
+
self.assertEqual(device.iid, "1219790537807339")
|
|
245
|
+
self.assertEqual(device.cdid, "demo-cdid")
|
|
246
|
+
self.assertEqual(device.openudid, "demo-openudid")
|
|
247
|
+
|
|
248
|
+
def test_mobile_device_pool_round_robin(self):
|
|
249
|
+
old_pool = DouyinMobileCrawler._device_pool
|
|
250
|
+
old_index = DouyinMobileCrawler._device_pool_index
|
|
251
|
+
try:
|
|
252
|
+
DouyinMobileCrawler._device_pool = [
|
|
253
|
+
DouyinMobileDevice(device_id="1", iid="11"),
|
|
254
|
+
DouyinMobileDevice(device_id="2", iid="22"),
|
|
255
|
+
DouyinMobileDevice(device_id="3", iid="33"),
|
|
256
|
+
]
|
|
257
|
+
DouyinMobileCrawler._device_pool_index = 0
|
|
258
|
+
|
|
259
|
+
picked = [DouyinMobileCrawler._next_pooled_device().device_id for _ in range(5)]
|
|
260
|
+
|
|
261
|
+
self.assertEqual(picked, ["1", "2", "3", "1", "2"])
|
|
262
|
+
finally:
|
|
263
|
+
DouyinMobileCrawler._device_pool = old_pool
|
|
264
|
+
DouyinMobileCrawler._device_pool_index = old_index
|
|
265
|
+
|
|
266
|
+
def test_parse_video_info_prefers_story_default_play_url_by_data_size(self):
|
|
267
|
+
video_data = {
|
|
268
|
+
"cover": {"url_list": ["https://cdn.example/thumb.jpg"]},
|
|
269
|
+
"bit_rate": [
|
|
270
|
+
{
|
|
271
|
+
"bit_rate": 0,
|
|
272
|
+
"duration": 9682,
|
|
273
|
+
"play_addr": {
|
|
274
|
+
"url_list": ["https://cdn.example/story-default.mp4"],
|
|
275
|
+
"width": 720,
|
|
276
|
+
"height": 1280,
|
|
277
|
+
"data_size": 7567631,
|
|
278
|
+
},
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
"bit_rate": 632,
|
|
282
|
+
"duration": 9682,
|
|
283
|
+
"play_addr": {
|
|
284
|
+
"url_list": ["https://cdn.example/720p.mp4"],
|
|
285
|
+
"width": 720,
|
|
286
|
+
"height": 1280,
|
|
287
|
+
"data_size": 783567,
|
|
288
|
+
},
|
|
289
|
+
},
|
|
290
|
+
],
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
info = parse_video_info(video_data)
|
|
294
|
+
|
|
295
|
+
self.assertEqual(info["video_url"], "https://cdn.example/story-default.mp4")
|
|
296
|
+
self.assertEqual(info["thumb_url"], "https://cdn.example/thumb.jpg")
|
|
297
|
+
self.assertEqual(info["duration"], 9682)
|
|
298
|
+
|
|
299
|
+
|
|
204
300
|
class TestPlatformUrlMatching(unittest.TestCase):
|
|
205
301
|
def test_supported_platform_url_formats(self):
|
|
206
302
|
parsehub = ParseHub()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|