douyin-dl-cli 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_mode/__init__.py +1 -0
- agent_mode/__main__.py +8 -0
- agent_mode/auth_cmds.py +178 -0
- agent_mode/capabilities.py +84 -0
- agent_mode/discovery_cmds.py +152 -0
- agent_mode/discovery_common.py +33 -0
- agent_mode/discovery_search.py +182 -0
- agent_mode/entry.py +299 -0
- agent_mode/envelope.py +235 -0
- agent_mode/events.py +113 -0
- agent_mode/inspect_cmd.py +130 -0
- agent_mode/logging_guard.py +29 -0
- agent_mode/login_cmd.py +204 -0
- agent_mode/login_import.py +128 -0
- agent_mode/media_cmds.py +49 -0
- agent_mode/media_live.py +277 -0
- agent_mode/media_live_recovery.py +174 -0
- agent_mode/media_transcribe.py +272 -0
- agent_mode/state.py +199 -0
- agent_mode/version.py +33 -0
- auth/__init__.py +4 -0
- auth/cookie_manager.py +75 -0
- auth/ms_token_conf.py +148 -0
- auth/ms_token_manager.py +236 -0
- cli/__init__.py +3 -0
- cli/login_flow.py +70 -0
- cli/main.py +530 -0
- cli/progress_display.py +314 -0
- cli/whisper_transcribe.py +533 -0
- config/__init__.py +4 -0
- config/config_loader.py +381 -0
- config/default_config.py +126 -0
- control/__init__.py +5 -0
- control/queue_manager.py +148 -0
- control/rate_limiter.py +28 -0
- control/retry_handler.py +41 -0
- core/__init__.py +15 -0
- core/api_client.py +2244 -0
- core/audio_extraction.py +381 -0
- core/comments_collector.py +349 -0
- core/discovery.py +127 -0
- core/downloader_base.py +1820 -0
- core/downloader_factory.py +85 -0
- core/ffmpeg.py +21 -0
- core/item_reasons.py +112 -0
- core/live_downloader.py +173 -0
- core/live_finalize.py +218 -0
- core/live_process.py +168 -0
- core/live_replay_downloader.py +349 -0
- core/live_session.py +300 -0
- core/live_streams.py +91 -0
- core/metadata.py +96 -0
- core/mix_downloader.py +183 -0
- core/music_downloader.py +243 -0
- core/output_naming.py +70 -0
- core/retry_executor.py +274 -0
- core/silent_audio.py +110 -0
- core/transcript_manager.py +424 -0
- core/url_parser.py +156 -0
- core/user_downloader.py +745 -0
- core/user_mode_registry.py +34 -0
- core/user_modes/__init__.py +17 -0
- core/user_modes/base_strategy.py +495 -0
- core/user_modes/collect_mix_strategy.py +67 -0
- core/user_modes/collect_strategy.py +158 -0
- core/user_modes/like_strategy.py +6 -0
- core/user_modes/mix_routing.py +49 -0
- core/user_modes/mix_strategy.py +84 -0
- core/user_modes/music_strategy.py +25 -0
- core/user_modes/post_strategy.py +486 -0
- core/user_modes/post_time_boundary.py +75 -0
- core/video_downloader.py +76 -0
- douyin_dl_cli-2.1.0.dist-info/METADATA +258 -0
- douyin_dl_cli-2.1.0.dist-info/RECORD +99 -0
- douyin_dl_cli-2.1.0.dist-info/WHEEL +5 -0
- douyin_dl_cli-2.1.0.dist-info/entry_points.txt +2 -0
- douyin_dl_cli-2.1.0.dist-info/licenses/LICENSE +21 -0
- douyin_dl_cli-2.1.0.dist-info/top_level.txt +10 -0
- server/__init__.py +13 -0
- server/app.py +215 -0
- server/jobs.py +163 -0
- storage/__init__.py +5 -0
- storage/database.py +815 -0
- storage/file_manager.py +563 -0
- storage/live_recordings.py +148 -0
- storage/metadata_handler.py +51 -0
- tools/__init__.py +1 -0
- tools/cookie_fetcher.py +390 -0
- utils/__init__.py +14 -0
- utils/abogus.py +865 -0
- utils/cookie_utils.py +45 -0
- utils/helpers.py +24 -0
- utils/logger.py +97 -0
- utils/naming.py +241 -0
- utils/naming_limits.py +116 -0
- utils/notifier.py +293 -0
- utils/paid_content.py +141 -0
- utils/validators.py +145 -0
- utils/xbogus.py +216 -0
agent_mode/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""douyin-dl agent 模式:给 AI agent 用的机器可读入口(CLI-only,不与桌面版同步)。"""
|
agent_mode/__main__.py
ADDED
agent_mode/auth_cmds.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""`doctor`:报告运行环境、状态目录、cookie 状态、依赖和各 verb 的可用性。
|
|
2
|
+
|
|
3
|
+
默认完全离线;加 `--online` 时用 profile/self 发 1 次请求验证登录态。
|
|
4
|
+
除 INTERNAL 外总是退出 0。输出里只有 cookie 的键名,没有值。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import datetime
|
|
9
|
+
import importlib.util
|
|
10
|
+
import platform
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from typing import Any, Dict, List, Optional
|
|
14
|
+
|
|
15
|
+
from agent_mode import state
|
|
16
|
+
from agent_mode.capabilities import URL_TYPE_CAPABILITIES, VERB_CAPABILITIES
|
|
17
|
+
from agent_mode.envelope import add_warning, clean_text, install_action, python_command
|
|
18
|
+
from agent_mode.version import get_version
|
|
19
|
+
|
|
20
|
+
SELF_PROFILE_ENDPOINT = "/aweme/v1/web/user/profile/self/"
|
|
21
|
+
SECONDS_PER_DAY = 86400
|
|
22
|
+
AGE_DAYS_PRECISION = 1
|
|
23
|
+
ONLINE_DETAIL_MAX_CHARS = 120
|
|
24
|
+
ONLINE_LOGGED_IN = "logged_in"
|
|
25
|
+
ONLINE_LOGIN_REQUIRED = "login_required"
|
|
26
|
+
ONLINE_EMPTY = "empty"
|
|
27
|
+
ONLINE_ERROR = "error"
|
|
28
|
+
ONLINE_SKIPPED = "skipped"
|
|
29
|
+
# verb -> 它额外依赖的本地组件(deps 里的键名)
|
|
30
|
+
VERB_DEPENDENCIES = {
|
|
31
|
+
"login": ("playwright",),
|
|
32
|
+
"live": ("ffmpeg",),
|
|
33
|
+
"transcribe": ("whisper",),
|
|
34
|
+
}
|
|
35
|
+
# 能力级别 -> (满足它的登录状态, 不满足时写进 blocked_by 的名字)
|
|
36
|
+
_LEVEL_REQUIREMENTS = {
|
|
37
|
+
"cookie": ((state.LOGIN_STATUS_DEVICE_ONLY, state.LOGIN_STATUS_LOGGED_IN), "cookies"),
|
|
38
|
+
"login": ((state.LOGIN_STATUS_LOGGED_IN,), "login"),
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def register(subparsers: Any) -> None:
|
|
43
|
+
parser = subparsers.add_parser(
|
|
44
|
+
"doctor", help="report environment, login state and which verbs are usable"
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--online",
|
|
48
|
+
action="store_true",
|
|
49
|
+
help="also verify the saved login with one request to Douyin",
|
|
50
|
+
)
|
|
51
|
+
parser.set_defaults(handler=run_doctor)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _state_dir_info() -> Dict[str, Any]:
|
|
55
|
+
path = state.state_dir()
|
|
56
|
+
return {"path": str(path), "exists": path.is_dir(), "mode": state.mode_string(path)}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _cookies_info(cookies: Dict[str, str]) -> Dict[str, Any]:
|
|
60
|
+
path = state.cookies_path()
|
|
61
|
+
exists = path.is_file()
|
|
62
|
+
mtime = None
|
|
63
|
+
age_days = None
|
|
64
|
+
if exists:
|
|
65
|
+
stamp = path.stat().st_mtime
|
|
66
|
+
mtime = datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).isoformat()
|
|
67
|
+
age_days = round(max(time.time() - stamp, 0) / SECONDS_PER_DAY, AGE_DAYS_PRECISION)
|
|
68
|
+
return {
|
|
69
|
+
"path": str(path),
|
|
70
|
+
"exists": exists,
|
|
71
|
+
"mode": state.mode_string(path),
|
|
72
|
+
"mtime": mtime,
|
|
73
|
+
"age_days": age_days,
|
|
74
|
+
"keys_present": state.keys_present(cookies),
|
|
75
|
+
"login_status": state.login_status(cookies),
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _module_dep(module: str, *pip_args: str, hint: str = "") -> Dict[str, Any]:
|
|
80
|
+
try:
|
|
81
|
+
available = importlib.util.find_spec(module) is not None
|
|
82
|
+
except (ImportError, ValueError):
|
|
83
|
+
available = False
|
|
84
|
+
info: Dict[str, Any] = {"available": available}
|
|
85
|
+
if not available:
|
|
86
|
+
info["install"] = install_action(*pip_args, hint=hint)
|
|
87
|
+
return info
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _ffmpeg_dep() -> Dict[str, Any]:
|
|
91
|
+
from core.ffmpeg import resolve_ffmpeg_path
|
|
92
|
+
|
|
93
|
+
path = resolve_ffmpeg_path()
|
|
94
|
+
info: Dict[str, Any] = {"available": bool(path), "path": path or None}
|
|
95
|
+
if not path:
|
|
96
|
+
info["install"] = install_action("imageio-ffmpeg==0.6.0")
|
|
97
|
+
return info
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _deps_info() -> Dict[str, Any]:
|
|
101
|
+
return {
|
|
102
|
+
"ffmpeg": _ffmpeg_dep(),
|
|
103
|
+
"playwright": _module_dep(
|
|
104
|
+
"playwright",
|
|
105
|
+
"playwright",
|
|
106
|
+
hint="Then run: %s" % python_command("-m", "playwright", "install", "chromium"),
|
|
107
|
+
),
|
|
108
|
+
"whisper": _module_dep("whisper", "openai-whisper"),
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _verb_blockers(verb: str, login_state: str, deps: Dict[str, Any]) -> List[str]:
|
|
113
|
+
blockers = []
|
|
114
|
+
requirement = _LEVEL_REQUIREMENTS.get(VERB_CAPABILITIES[verb])
|
|
115
|
+
if requirement is not None and login_state not in requirement[0]:
|
|
116
|
+
blockers.append(requirement[1])
|
|
117
|
+
for dep in VERB_DEPENDENCIES.get(verb, ()):
|
|
118
|
+
if not deps[dep]["available"]:
|
|
119
|
+
blockers.append(dep)
|
|
120
|
+
return blockers
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _capabilities_info(login_state: str, deps: Dict[str, Any]) -> Dict[str, Any]:
|
|
124
|
+
verbs = {}
|
|
125
|
+
for verb, level in VERB_CAPABILITIES.items():
|
|
126
|
+
blockers = _verb_blockers(verb, login_state, deps)
|
|
127
|
+
verbs[verb] = {"level": level, "ready": not blockers, "blocked_by": blockers}
|
|
128
|
+
return {"verbs": verbs, "url_types": dict(URL_TYPE_CAPABILITIES)}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _online_result(status: str, detail: str = "") -> Dict[str, Any]:
|
|
132
|
+
return {
|
|
133
|
+
"endpoint": SELF_PROFILE_ENDPOINT,
|
|
134
|
+
"status": status,
|
|
135
|
+
"logged_in": status == ONLINE_LOGGED_IN,
|
|
136
|
+
"detail": clean_text(detail, ONLINE_DETAIL_MAX_CHARS),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
async def _online_check(cookies: Dict[str, str]) -> Dict[str, Any]:
|
|
141
|
+
"""用 profile/self 验证登录态。没有会话 cookie 时不发请求。"""
|
|
142
|
+
if state.login_status(cookies) != state.LOGIN_STATUS_LOGGED_IN:
|
|
143
|
+
return _online_result(ONLINE_SKIPPED, "no session cookie saved; nothing to verify")
|
|
144
|
+
|
|
145
|
+
from core.api_client import DouyinAPIClient, LoginRequiredError
|
|
146
|
+
|
|
147
|
+
proxy = state.build_config().get("proxy")
|
|
148
|
+
try:
|
|
149
|
+
async with DouyinAPIClient(cookies, proxy=proxy) as client:
|
|
150
|
+
user = await client.get_self_info()
|
|
151
|
+
except LoginRequiredError:
|
|
152
|
+
return _online_result(ONLINE_LOGIN_REQUIRED, "Douyin rejected the saved session")
|
|
153
|
+
except Exception as exc: # noqa: BLE001 — doctor 只报告,不因联网失败而失败
|
|
154
|
+
return _online_result(ONLINE_ERROR, type(exc).__name__)
|
|
155
|
+
if user:
|
|
156
|
+
return _online_result(ONLINE_LOGGED_IN)
|
|
157
|
+
return _online_result(ONLINE_EMPTY, "empty response (rate limit, anti-bot or network)")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def run_doctor(args: argparse.Namespace) -> Dict[str, Any]:
|
|
161
|
+
cookies = state.read_cookies()
|
|
162
|
+
for warning in state.permission_warnings():
|
|
163
|
+
add_warning(warning)
|
|
164
|
+
deps = _deps_info()
|
|
165
|
+
online: Optional[Dict[str, Any]] = None
|
|
166
|
+
if args.online:
|
|
167
|
+
online = await _online_check(cookies)
|
|
168
|
+
return {
|
|
169
|
+
"version": get_version(),
|
|
170
|
+
"python": platform.python_version(),
|
|
171
|
+
"platform": platform.platform(),
|
|
172
|
+
"executable": sys.executable,
|
|
173
|
+
"state_dir": _state_dir_info(),
|
|
174
|
+
"cookies": _cookies_info(cookies),
|
|
175
|
+
"deps": deps,
|
|
176
|
+
"capabilities": _capabilities_info(state.login_status(cookies), deps),
|
|
177
|
+
"online_check": online,
|
|
178
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""能力表:哪些端点受门禁、每种 url_type 和每个 verb 需要什么级别。
|
|
2
|
+
|
|
3
|
+
`tests/test_agent_capabilities_drift.py` 会把这里的表与 `core/api_client.py`、
|
|
4
|
+
`utils/validators.py` 的实际情况对照,共享代码变了而这里没跟上时测试会失败。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
ANONYMOUS = "anonymous"
|
|
8
|
+
COOKIE = "cookie"
|
|
9
|
+
LOGIN = "login"
|
|
10
|
+
LOCAL = "local"
|
|
11
|
+
ARGUS_GATED = "argus_gated"
|
|
12
|
+
UNSUPPORTED = "unsupported"
|
|
13
|
+
LEVELS = (ANONYMOUS, COOKIE, LOGIN, LOCAL, ARGUS_GATED, UNSUPPORTED)
|
|
14
|
+
|
|
15
|
+
# core/api_client.py 中经 `_request_json_gated` 发出的全部路径。
|
|
16
|
+
# 开源 CLI 没有 page bridge,这些请求会被 ArgusSecurityPlugin 确定性拒绝。
|
|
17
|
+
GATED_ENDPOINTS = frozenset(
|
|
18
|
+
{
|
|
19
|
+
"/aweme/v1/web/aweme/detail/",
|
|
20
|
+
"/aweme/v1/web/aweme/post/",
|
|
21
|
+
"/aweme/v1/web/aweme/favorite/",
|
|
22
|
+
"/aweme/v1/web/aweme/listcollection/",
|
|
23
|
+
"/aweme/v1/web/mix/list/",
|
|
24
|
+
"/aweme/v1/web/mix/detail/",
|
|
25
|
+
"/aweme/v1/web/mix/aweme/",
|
|
26
|
+
"/aweme/v1/web/mix/listcollection/",
|
|
27
|
+
"/aweme/v1/web/series/list/",
|
|
28
|
+
"/aweme/v1/web/music/list/",
|
|
29
|
+
"/aweme/v1/web/music/detail/",
|
|
30
|
+
"/aweme/v1/web/music/aweme/",
|
|
31
|
+
"/aweme/v1/web/collects/list/",
|
|
32
|
+
"/aweme/v1/web/collects/video/list/",
|
|
33
|
+
}
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
VERB_CAPABILITIES = {
|
|
37
|
+
"doctor": LOCAL,
|
|
38
|
+
"inspect": ANONYMOUS,
|
|
39
|
+
"hot": ANONYMOUS,
|
|
40
|
+
"search": LOGIN,
|
|
41
|
+
"user": COOKIE,
|
|
42
|
+
"login": LOCAL,
|
|
43
|
+
"live": ANONYMOUS,
|
|
44
|
+
"transcribe": LOCAL,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# `parse_url_type` 对未解析短链返回的占位类型;inspect 先解析短链,它不进能力表。
|
|
48
|
+
UNRESOLVED_URL_TYPE = "short"
|
|
49
|
+
|
|
50
|
+
# url_type(取值以 utils/validators.parse_url_type 为准)-> 能力级别
|
|
51
|
+
URL_TYPE_CAPABILITIES = {
|
|
52
|
+
"video": ARGUS_GATED,
|
|
53
|
+
"gallery": ARGUS_GATED,
|
|
54
|
+
"collection": ARGUS_GATED,
|
|
55
|
+
"music": ARGUS_GATED,
|
|
56
|
+
"user": COOKIE,
|
|
57
|
+
"live": ANONYMOUS,
|
|
58
|
+
"live_replay": UNSUPPORTED,
|
|
59
|
+
"lvdetail": UNSUPPORTED,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
URL_TYPE_SUGGESTED_VERB = {
|
|
63
|
+
"user": "user",
|
|
64
|
+
"live": "live",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
URL_TYPE_NOTES = {
|
|
68
|
+
"video": "Single-video download needs the Argus-gated aweme/detail endpoint.",
|
|
69
|
+
"gallery": "Image-post download needs the Argus-gated aweme/detail endpoint.",
|
|
70
|
+
"collection": "Collection (mix) download needs the Argus-gated mix endpoints.",
|
|
71
|
+
"music": "Music download needs the Argus-gated music endpoints.",
|
|
72
|
+
"user": (
|
|
73
|
+
"Profile info is available via `user`. Downloading this user's posts is "
|
|
74
|
+
"Argus-gated and not offered in agent mode."
|
|
75
|
+
),
|
|
76
|
+
"live": "A live room can be recorded with `live <url> --max-duration S` while it is on air.",
|
|
77
|
+
"live_replay": "Live replays are not offered in agent mode v1.",
|
|
78
|
+
"lvdetail": "Douyin cinema (lvdetail) content is DRM-encrypted and cannot be downloaded.",
|
|
79
|
+
}
|
|
80
|
+
UNKNOWN_URL_NOTE = "Not a recognised Douyin URL."
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def url_type_level(url_type: object) -> str:
|
|
84
|
+
return URL_TYPE_CAPABILITIES.get(str(url_type or ""), UNSUPPORTED)
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""`hot`、`search`、`user`:联网只读 verb,只输出投影字段,绝不含媒体直链。接口见 entry.py。
|
|
2
|
+
|
|
3
|
+
`search` 的实现在 discovery_search.py,这里只负责注册。
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import re
|
|
8
|
+
from typing import Any, Dict
|
|
9
|
+
from urllib.parse import quote
|
|
10
|
+
|
|
11
|
+
from agent_mode import discovery_search, state
|
|
12
|
+
from agent_mode.capabilities import URL_TYPE_NOTES
|
|
13
|
+
from agent_mode.discovery_common import (
|
|
14
|
+
ID_MAX_CHARS,
|
|
15
|
+
RETRY_AFTER_S,
|
|
16
|
+
as_dict,
|
|
17
|
+
as_int,
|
|
18
|
+
open_client,
|
|
19
|
+
upstream_empty,
|
|
20
|
+
)
|
|
21
|
+
from agent_mode.entry import bounded_int
|
|
22
|
+
from agent_mode.envelope import DESC_MAX_CHARS, NAME_MAX_CHARS, AgentError, clean_text
|
|
23
|
+
|
|
24
|
+
HOT_DEFAULT_MAX = 50
|
|
25
|
+
HOT_MAX_LIMIT = 100
|
|
26
|
+
SEARCH_URL_PREFIX = "https://www.douyin.com/search/"
|
|
27
|
+
PUBLISH_DAYS_HELP = "only posts published within N days (0 = any time)"
|
|
28
|
+
# sec_uid 都以这个前缀开头;抖音号、数字 uid 等其他裸 id 不能拿去查主页
|
|
29
|
+
SEC_UID_PREFIX = "MS4wLjAB"
|
|
30
|
+
SEC_UID_HINT = "pass the profile URL or a sec_uid (starts with %s)" % SEC_UID_PREFIX
|
|
31
|
+
_SEC_UID_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
32
|
+
_HOT_OPTIONAL_FIELDS = ("label", "video_count")
|
|
33
|
+
_USER_STAT_FIELDS = ("follower_count", "following_count", "aweme_count", "total_favorited")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def register(subparsers: Any) -> None:
|
|
37
|
+
hot = subparsers.add_parser("hot", help="list the Douyin hot-search board")
|
|
38
|
+
hot.add_argument(
|
|
39
|
+
"--max", type=bounded_int(1, HOT_MAX_LIMIT), default=HOT_DEFAULT_MAX, metavar="N"
|
|
40
|
+
)
|
|
41
|
+
hot.set_defaults(handler=run_hot)
|
|
42
|
+
search = subparsers.add_parser("search", help="search posts by keyword (needs login)")
|
|
43
|
+
search.add_argument("keyword")
|
|
44
|
+
search.add_argument(
|
|
45
|
+
"--max",
|
|
46
|
+
type=bounded_int(1, discovery_search.SEARCH_MAX_LIMIT),
|
|
47
|
+
default=discovery_search.SEARCH_DEFAULT_MAX,
|
|
48
|
+
metavar="N",
|
|
49
|
+
)
|
|
50
|
+
search.add_argument(
|
|
51
|
+
"--sort", choices=sorted(discovery_search.SEARCH_SORT_TYPES), default="general"
|
|
52
|
+
)
|
|
53
|
+
search.add_argument(
|
|
54
|
+
"--publish-days",
|
|
55
|
+
type=int,
|
|
56
|
+
choices=discovery_search.SEARCH_PUBLISH_DAYS,
|
|
57
|
+
default=0,
|
|
58
|
+
help=PUBLISH_DAYS_HELP,
|
|
59
|
+
)
|
|
60
|
+
search.set_defaults(handler=discovery_search.run_search)
|
|
61
|
+
user = subparsers.add_parser("user", help="show a user's public profile (needs cookies)")
|
|
62
|
+
user.add_argument("target", help="profile URL or sec_uid")
|
|
63
|
+
user.set_defaults(handler=run_user)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _project_hot_item(raw: Dict[str, Any], rank: int) -> Dict[str, Any]:
|
|
67
|
+
word = clean_text(raw.get("word"), NAME_MAX_CHARS)
|
|
68
|
+
item = {"rank": rank, "word": word, "hot_value": as_int(raw.get("hot_value"))}
|
|
69
|
+
for field in _HOT_OPTIONAL_FIELDS:
|
|
70
|
+
value = as_int(raw.get(field))
|
|
71
|
+
if value:
|
|
72
|
+
item[field] = value
|
|
73
|
+
item["search_url"] = SEARCH_URL_PREFIX + quote(word, safe="")
|
|
74
|
+
return item
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def run_hot(args: argparse.Namespace) -> Dict[str, Any]:
|
|
78
|
+
from core.api_client import LoginRequiredError
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
async with open_client(state.read_cookies()) as client:
|
|
82
|
+
page = await client.get_hot_search_board()
|
|
83
|
+
except LoginRequiredError:
|
|
84
|
+
raise AgentError("LOGIN_REQUIRED", "Douyin asked for a login for the hot board.") from None
|
|
85
|
+
rows = [raw for raw in page.get("items") or [] if clean_text(as_dict(raw).get("word"))]
|
|
86
|
+
items = [_project_hot_item(raw, rank) for rank, raw in enumerate(rows[: args.max], 1)]
|
|
87
|
+
if not items:
|
|
88
|
+
empty = {"items": [], "count": 0}
|
|
89
|
+
raise upstream_empty("Douyin returned an empty hot board.", RETRY_AFTER_S, empty)
|
|
90
|
+
return {"items": items, "count": len(items)}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _not_a_profile() -> AgentError:
|
|
94
|
+
action = {"kind": "fix_usage", "hint": SEC_UID_HINT}
|
|
95
|
+
return AgentError("USAGE", "Expected a Douyin profile URL or a sec_uid.", action=action)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def _resolve_sec_uid(target: str) -> str:
|
|
99
|
+
"""裸 sec_uid 直接用;否则按 URL(含短链、分享文案)解析出 sec_uid。"""
|
|
100
|
+
from agent_mode.inspect_cmd import resolve_input_url
|
|
101
|
+
from core.url_parser import URLParser
|
|
102
|
+
|
|
103
|
+
bare = target.strip()
|
|
104
|
+
if _SEC_UID_RE.match(bare):
|
|
105
|
+
if not bare.startswith(SEC_UID_PREFIX):
|
|
106
|
+
# 抖音号之类的裸 id 查不到主页,不发请求
|
|
107
|
+
raise _not_a_profile()
|
|
108
|
+
return bare
|
|
109
|
+
url = await resolve_input_url(target)
|
|
110
|
+
if url is None:
|
|
111
|
+
raise upstream_empty("Could not resolve the link.", RETRY_AFTER_S, {"sec_uid": None})
|
|
112
|
+
parsed = URLParser.parse(url) or {}
|
|
113
|
+
sec_uid = str(parsed.get("sec_uid") or "") if parsed.get("type") == "user" else ""
|
|
114
|
+
if not _SEC_UID_RE.match(sec_uid):
|
|
115
|
+
raise _not_a_profile()
|
|
116
|
+
return sec_uid
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _project_user(raw: Dict[str, Any], sec_uid: str) -> Dict[str, Any]:
|
|
120
|
+
data: Dict[str, Any] = {
|
|
121
|
+
"sec_uid": clean_text(raw.get("sec_uid") or sec_uid, ID_MAX_CHARS),
|
|
122
|
+
"uid": clean_text(raw.get("uid"), ID_MAX_CHARS),
|
|
123
|
+
"unique_id": clean_text(raw.get("unique_id") or raw.get("short_id"), NAME_MAX_CHARS),
|
|
124
|
+
"nickname": clean_text(raw.get("nickname"), NAME_MAX_CHARS),
|
|
125
|
+
"signature": clean_text(raw.get("signature"), DESC_MAX_CHARS),
|
|
126
|
+
}
|
|
127
|
+
for field in _USER_STAT_FIELDS:
|
|
128
|
+
data[field] = as_int(raw.get(field))
|
|
129
|
+
ip_location = clean_text(raw.get("ip_location"), NAME_MAX_CHARS)
|
|
130
|
+
if ip_location:
|
|
131
|
+
data["ip_location"] = ip_location
|
|
132
|
+
data["note"] = URL_TYPE_NOTES["user"]
|
|
133
|
+
return data
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
async def run_user(args: argparse.Namespace) -> Dict[str, Any]:
|
|
137
|
+
cookies = state.read_cookies()
|
|
138
|
+
if not cookies:
|
|
139
|
+
# 没有 cookie 文件时 profile 接口拿不到数据,不发任何请求(含短链解析)
|
|
140
|
+
raise AgentError("LOGIN_REQUIRED", "The user verb needs saved cookies; none are saved.")
|
|
141
|
+
|
|
142
|
+
from core.api_client import LoginRequiredError
|
|
143
|
+
|
|
144
|
+
sec_uid = await _resolve_sec_uid(args.target)
|
|
145
|
+
try:
|
|
146
|
+
async with open_client(cookies) as client:
|
|
147
|
+
raw = await client.get_user_info(sec_uid)
|
|
148
|
+
except LoginRequiredError:
|
|
149
|
+
raise AgentError("LOGIN_REQUIRED", "Douyin asked for a login for this profile.") from None
|
|
150
|
+
if not isinstance(raw, dict) or not raw:
|
|
151
|
+
raise upstream_empty("Douyin returned no profile.", RETRY_AFTER_S, {"sec_uid": sec_uid})
|
|
152
|
+
return _project_user(raw, sec_uid)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""`hot`、`search`、`user` 共用的常量与小工具。"""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, Optional
|
|
4
|
+
|
|
5
|
+
from agent_mode import state
|
|
6
|
+
from agent_mode.envelope import AgentError, retry_action
|
|
7
|
+
|
|
8
|
+
RETRY_AFTER_S = 60
|
|
9
|
+
ID_MAX_CHARS = 128
|
|
10
|
+
EMPTY_HINT = "Likely a rate limit or anti-bot response. Wait before retrying; do not loop."
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def as_int(value: Any) -> Optional[int]:
|
|
14
|
+
"""平台数值可能是 int、数字字符串或缺失;解析不了就是 None。"""
|
|
15
|
+
try:
|
|
16
|
+
return None if isinstance(value, bool) else int(value)
|
|
17
|
+
except (TypeError, ValueError):
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def as_dict(value: Any) -> Dict[str, Any]:
|
|
22
|
+
return value if isinstance(value, dict) else {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def open_client(cookies: Dict[str, str]) -> Any:
|
|
26
|
+
from core.api_client import DouyinAPIClient
|
|
27
|
+
|
|
28
|
+
return DouyinAPIClient(cookies, proxy=state.build_config().get("proxy"))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def upstream_empty(message: str, retry_after_s: int, data: Dict[str, Any]) -> AgentError:
|
|
32
|
+
action = retry_action(retry_after_s, EMPTY_HINT)
|
|
33
|
+
return AgentError("UPSTREAM_EMPTY", message, action=action, data=data)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""`search` 的实现:翻页、投影、区分「确实没有结果」和「请求失败或被拦」。"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import datetime
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from agent_mode import state
|
|
9
|
+
from agent_mode.discovery_common import (
|
|
10
|
+
EMPTY_HINT,
|
|
11
|
+
ID_MAX_CHARS,
|
|
12
|
+
as_dict,
|
|
13
|
+
as_int,
|
|
14
|
+
open_client,
|
|
15
|
+
upstream_empty,
|
|
16
|
+
)
|
|
17
|
+
from agent_mode.envelope import DESC_MAX_CHARS, NAME_MAX_CHARS, AgentError, clean_text, retry_action
|
|
18
|
+
|
|
19
|
+
SEARCH_DEFAULT_MAX = 20
|
|
20
|
+
SEARCH_MAX_LIMIT = 50
|
|
21
|
+
# 与 DouyinAPIClient.search_aweme 的 sort_type / publish_time 取值对应
|
|
22
|
+
SEARCH_SORT_TYPES = {"general": 0, "likes": 1, "latest": 2}
|
|
23
|
+
SEARCH_PUBLISH_DAYS = (0, 1, 7, 182)
|
|
24
|
+
SEARCH_PAGE_SIZE = 10
|
|
25
|
+
# 翻页间隔比人用 CLI 保守,避免 agent 连续翻页触发风控
|
|
26
|
+
SEARCH_PAGE_INTERVAL_S = 1.0
|
|
27
|
+
SEARCH_RETRY_AFTER_S = 120
|
|
28
|
+
KEYWORD_MAX_CHARS = 100
|
|
29
|
+
MS_PER_SECOND = 1000
|
|
30
|
+
DURATION_PRECISION = 1
|
|
31
|
+
ISO_UTC_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
|
32
|
+
UPSTREAM_OK_STATUS = 0
|
|
33
|
+
# 状态码为 0 但其实被风控拦下时,响应里 search_nil_info.search_nil_type 的取值
|
|
34
|
+
BLOCKED_NIL_TYPES = ("verify_check", "hit_shark")
|
|
35
|
+
# 与 core/downloader_base.py 的 BaseDownloader._GALLERY_AWEME_TYPES 一致(有漂移测试)
|
|
36
|
+
GALLERY_AWEME_TYPES = (2, 68, 150)
|
|
37
|
+
# media_type -> 分享链接的路径段
|
|
38
|
+
SHARE_URL_TEMPLATE = "https://www.douyin.com/%s/%s"
|
|
39
|
+
SHARE_URL_SEGMENTS = {"video": "video", "gallery": "note"}
|
|
40
|
+
_STAT_FIELDS = ("digg", "comment", "share", "collect")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _media_type(raw: Dict[str, Any]) -> str:
|
|
44
|
+
image_post = as_dict(raw.get("image_post_info"))
|
|
45
|
+
has_images = raw.get("images") or image_post.get("images") or image_post.get("image_list")
|
|
46
|
+
return "gallery" if has_images or raw.get("aweme_type") in GALLERY_AWEME_TYPES else "video"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _iso_utc(epoch: Any) -> Optional[str]:
|
|
50
|
+
seconds = as_int(epoch) or 0
|
|
51
|
+
try:
|
|
52
|
+
moment = datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc)
|
|
53
|
+
except (OverflowError, OSError, ValueError):
|
|
54
|
+
return None
|
|
55
|
+
return moment.strftime(ISO_UTC_FORMAT) if seconds > 0 else None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _duration_s(raw: Dict[str, Any]) -> Optional[float]:
|
|
59
|
+
millis = as_int(as_dict(raw.get("video")).get("duration")) or as_int(raw.get("duration"))
|
|
60
|
+
return round(millis / MS_PER_SECOND, DURATION_PRECISION) if millis and millis > 0 else None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _project_aweme(raw: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
64
|
+
"""只挑契约里的字段;video / images 等带直链的子树整个不输出。"""
|
|
65
|
+
aweme_id = clean_text(raw.get("aweme_id"), ID_MAX_CHARS)
|
|
66
|
+
if not aweme_id.isdigit():
|
|
67
|
+
return None
|
|
68
|
+
author = as_dict(raw.get("author"))
|
|
69
|
+
stats = as_dict(raw.get("statistics"))
|
|
70
|
+
media_type = _media_type(raw)
|
|
71
|
+
return {
|
|
72
|
+
"aweme_id": aweme_id,
|
|
73
|
+
"desc": clean_text(raw.get("desc"), DESC_MAX_CHARS),
|
|
74
|
+
"author": {
|
|
75
|
+
"nickname": clean_text(author.get("nickname"), NAME_MAX_CHARS),
|
|
76
|
+
"sec_uid": clean_text(author.get("sec_uid"), ID_MAX_CHARS),
|
|
77
|
+
},
|
|
78
|
+
"stats": {name: as_int(stats.get(name + "_count")) for name in _STAT_FIELDS},
|
|
79
|
+
"create_time": _iso_utc(raw.get("create_time")),
|
|
80
|
+
"duration_s": _duration_s(raw),
|
|
81
|
+
"media_type": media_type,
|
|
82
|
+
"share_url": SHARE_URL_TEMPLATE % (SHARE_URL_SEGMENTS[media_type], aweme_id),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _absorb_page(page: Dict[str, Any], items: List[Dict[str, Any]], limit: int) -> bool:
|
|
87
|
+
"""把一页结果去重后并入 items;返回这一页是否还有没放进去的新条目。"""
|
|
88
|
+
seen = {item["aweme_id"] for item in items}
|
|
89
|
+
for raw in page.get("items") or []:
|
|
90
|
+
projected = _project_aweme(as_dict(raw))
|
|
91
|
+
if projected is None or projected["aweme_id"] in seen:
|
|
92
|
+
continue
|
|
93
|
+
if len(items) >= limit:
|
|
94
|
+
return True
|
|
95
|
+
seen.add(projected["aweme_id"])
|
|
96
|
+
items.append(projected)
|
|
97
|
+
return False
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class _PageFailed(Exception):
|
|
101
|
+
"""某一页请求失败或被拦;调用方按已拿到的条数决定错误码。"""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _is_failed_page(page: Dict[str, Any]) -> bool:
|
|
105
|
+
"""空页是请求失败或被拦,而不是确实没有结果(或自然翻到底)。
|
|
106
|
+
|
|
107
|
+
只有上游明确报成功(响应体里 status_code 为 0)且没有风控标记,才算真的没有结果。
|
|
108
|
+
"""
|
|
109
|
+
if page.get("items"):
|
|
110
|
+
return False
|
|
111
|
+
raw = as_dict(page.get("raw"))
|
|
112
|
+
if page.get("status_code") or as_int(raw.get("status_code")) != UPSTREAM_OK_STATUS:
|
|
113
|
+
return True
|
|
114
|
+
nil_type = as_dict(raw.get("search_nil_info")).get("search_nil_type")
|
|
115
|
+
return nil_type in BLOCKED_NIL_TYPES
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
async def _collect_search(client: Any, args: argparse.Namespace, items: List[Any]) -> bool:
|
|
119
|
+
"""翻页直到凑够 args.max 或到底,结果就地写进 items;返回 has_more。"""
|
|
120
|
+
offset = 0
|
|
121
|
+
while True:
|
|
122
|
+
page = await client.search_aweme(
|
|
123
|
+
args.keyword,
|
|
124
|
+
offset=offset,
|
|
125
|
+
count=SEARCH_PAGE_SIZE,
|
|
126
|
+
sort_type=SEARCH_SORT_TYPES[args.sort],
|
|
127
|
+
publish_time=args.publish_days,
|
|
128
|
+
)
|
|
129
|
+
if _is_failed_page(page):
|
|
130
|
+
raise _PageFailed("empty response at offset %d" % offset)
|
|
131
|
+
leftover = _absorb_page(page, items, args.max)
|
|
132
|
+
has_more = bool(page.get("has_more"))
|
|
133
|
+
if len(items) >= args.max:
|
|
134
|
+
return leftover or has_more
|
|
135
|
+
next_offset = as_int(page.get("max_cursor")) or 0
|
|
136
|
+
if not has_more or not page.get("items") or next_offset == offset:
|
|
137
|
+
return False
|
|
138
|
+
offset = next_offset
|
|
139
|
+
await asyncio.sleep(SEARCH_PAGE_INTERVAL_S)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _search_data(keyword: str, items: List[Dict[str, Any]], has_more: bool) -> Dict[str, Any]:
|
|
143
|
+
keyword = clean_text(keyword, KEYWORD_MAX_CHARS)
|
|
144
|
+
return {"keyword": keyword, "items": items, "count": len(items), "has_more": has_more}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _search_failure(args: argparse.Namespace, items: List[Any], reason: str) -> AgentError:
|
|
148
|
+
if not items:
|
|
149
|
+
empty = _search_data(args.keyword, [], False)
|
|
150
|
+
return upstream_empty("Search returned no items.", SEARCH_RETRY_AFTER_S, empty)
|
|
151
|
+
return AgentError(
|
|
152
|
+
"PARTIAL",
|
|
153
|
+
"Search stopped after %d of %d requested items (%s)." % (len(items), args.max, reason),
|
|
154
|
+
action=retry_action(SEARCH_RETRY_AFTER_S, EMPTY_HINT),
|
|
155
|
+
data=_search_data(args.keyword, items, True),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
async def run_search(args: argparse.Namespace) -> Dict[str, Any]:
|
|
160
|
+
cookies = state.read_cookies()
|
|
161
|
+
if state.login_status(cookies) != state.LOGIN_STATUS_LOGGED_IN:
|
|
162
|
+
# 没有会话 cookie 时搜索必然被拒,不构造 client,也不发请求
|
|
163
|
+
raise AgentError("LOGIN_REQUIRED", "Search needs a logged-in session; none is saved.")
|
|
164
|
+
|
|
165
|
+
from core.api_client import LoginRequiredError
|
|
166
|
+
|
|
167
|
+
items: List[Dict[str, Any]] = []
|
|
168
|
+
try:
|
|
169
|
+
async with open_client(cookies) as client:
|
|
170
|
+
has_more = await _collect_search(client, args, items)
|
|
171
|
+
except LoginRequiredError:
|
|
172
|
+
partial = _search_data(args.keyword, items, True) if items else None
|
|
173
|
+
message = "Douyin rejected the saved session for search."
|
|
174
|
+
raise AgentError("LOGIN_REQUIRED", message, data=partial) from None
|
|
175
|
+
except _PageFailed as exc:
|
|
176
|
+
raise _search_failure(args, items, str(exc)) from None
|
|
177
|
+
except Exception as exc: # noqa: BLE001 — 已有部分结果时带着返回,否则交给入口报 INTERNAL
|
|
178
|
+
if not items:
|
|
179
|
+
raise
|
|
180
|
+
raise _search_failure(args, items, type(exc).__name__) from None
|
|
181
|
+
# 走到这里而 items 为空:上游明确成功,只是这个关键词没有结果
|
|
182
|
+
return _search_data(args.keyword, items, has_more)
|