douyin-dl-cli 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. agent_mode/__init__.py +1 -0
  2. agent_mode/__main__.py +8 -0
  3. agent_mode/auth_cmds.py +178 -0
  4. agent_mode/capabilities.py +84 -0
  5. agent_mode/discovery_cmds.py +152 -0
  6. agent_mode/discovery_common.py +33 -0
  7. agent_mode/discovery_search.py +182 -0
  8. agent_mode/entry.py +299 -0
  9. agent_mode/envelope.py +235 -0
  10. agent_mode/events.py +113 -0
  11. agent_mode/inspect_cmd.py +130 -0
  12. agent_mode/logging_guard.py +29 -0
  13. agent_mode/login_cmd.py +204 -0
  14. agent_mode/login_import.py +128 -0
  15. agent_mode/media_cmds.py +49 -0
  16. agent_mode/media_live.py +277 -0
  17. agent_mode/media_live_recovery.py +174 -0
  18. agent_mode/media_transcribe.py +272 -0
  19. agent_mode/state.py +199 -0
  20. agent_mode/version.py +33 -0
  21. auth/__init__.py +4 -0
  22. auth/cookie_manager.py +75 -0
  23. auth/ms_token_conf.py +148 -0
  24. auth/ms_token_manager.py +236 -0
  25. cli/__init__.py +3 -0
  26. cli/login_flow.py +70 -0
  27. cli/main.py +530 -0
  28. cli/progress_display.py +314 -0
  29. cli/whisper_transcribe.py +533 -0
  30. config/__init__.py +4 -0
  31. config/config_loader.py +381 -0
  32. config/default_config.py +126 -0
  33. control/__init__.py +5 -0
  34. control/queue_manager.py +148 -0
  35. control/rate_limiter.py +28 -0
  36. control/retry_handler.py +41 -0
  37. core/__init__.py +15 -0
  38. core/api_client.py +2244 -0
  39. core/audio_extraction.py +381 -0
  40. core/comments_collector.py +349 -0
  41. core/discovery.py +127 -0
  42. core/downloader_base.py +1820 -0
  43. core/downloader_factory.py +85 -0
  44. core/ffmpeg.py +21 -0
  45. core/item_reasons.py +112 -0
  46. core/live_downloader.py +173 -0
  47. core/live_finalize.py +218 -0
  48. core/live_process.py +168 -0
  49. core/live_replay_downloader.py +349 -0
  50. core/live_session.py +300 -0
  51. core/live_streams.py +91 -0
  52. core/metadata.py +96 -0
  53. core/mix_downloader.py +183 -0
  54. core/music_downloader.py +243 -0
  55. core/output_naming.py +70 -0
  56. core/retry_executor.py +274 -0
  57. core/silent_audio.py +110 -0
  58. core/transcript_manager.py +424 -0
  59. core/url_parser.py +156 -0
  60. core/user_downloader.py +745 -0
  61. core/user_mode_registry.py +34 -0
  62. core/user_modes/__init__.py +17 -0
  63. core/user_modes/base_strategy.py +495 -0
  64. core/user_modes/collect_mix_strategy.py +67 -0
  65. core/user_modes/collect_strategy.py +158 -0
  66. core/user_modes/like_strategy.py +6 -0
  67. core/user_modes/mix_routing.py +49 -0
  68. core/user_modes/mix_strategy.py +84 -0
  69. core/user_modes/music_strategy.py +25 -0
  70. core/user_modes/post_strategy.py +486 -0
  71. core/user_modes/post_time_boundary.py +75 -0
  72. core/video_downloader.py +76 -0
  73. douyin_dl_cli-2.1.0.dist-info/METADATA +258 -0
  74. douyin_dl_cli-2.1.0.dist-info/RECORD +99 -0
  75. douyin_dl_cli-2.1.0.dist-info/WHEEL +5 -0
  76. douyin_dl_cli-2.1.0.dist-info/entry_points.txt +2 -0
  77. douyin_dl_cli-2.1.0.dist-info/licenses/LICENSE +21 -0
  78. douyin_dl_cli-2.1.0.dist-info/top_level.txt +10 -0
  79. server/__init__.py +13 -0
  80. server/app.py +215 -0
  81. server/jobs.py +163 -0
  82. storage/__init__.py +5 -0
  83. storage/database.py +815 -0
  84. storage/file_manager.py +563 -0
  85. storage/live_recordings.py +148 -0
  86. storage/metadata_handler.py +51 -0
  87. tools/__init__.py +1 -0
  88. tools/cookie_fetcher.py +390 -0
  89. utils/__init__.py +14 -0
  90. utils/abogus.py +865 -0
  91. utils/cookie_utils.py +45 -0
  92. utils/helpers.py +24 -0
  93. utils/logger.py +97 -0
  94. utils/naming.py +241 -0
  95. utils/naming_limits.py +116 -0
  96. utils/notifier.py +293 -0
  97. utils/paid_content.py +141 -0
  98. utils/validators.py +145 -0
  99. utils/xbogus.py +216 -0
agent_mode/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """douyin-dl agent 模式:给 AI agent 用的机器可读入口(CLI-only,不与桌面版同步)。"""
agent_mode/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ """`python -m agent_mode <verb> ...` 入口。"""
2
+
3
+ import sys
4
+
5
+ from agent_mode.entry import main
6
+
7
+ if __name__ == "__main__":
8
+ sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,178 @@
1
+ """`doctor`:报告运行环境、状态目录、cookie 状态、依赖和各 verb 的可用性。
2
+
3
+ 默认完全离线;加 `--online` 时用 profile/self 发 1 次请求验证登录态。
4
+ 除 INTERNAL 外总是退出 0。输出里只有 cookie 的键名,没有值。
5
+ """
6
+
7
+ import argparse
8
+ import datetime
9
+ import importlib.util
10
+ import platform
11
+ import sys
12
+ import time
13
+ from typing import Any, Dict, List, Optional
14
+
15
+ from agent_mode import state
16
+ from agent_mode.capabilities import URL_TYPE_CAPABILITIES, VERB_CAPABILITIES
17
+ from agent_mode.envelope import add_warning, clean_text, install_action, python_command
18
+ from agent_mode.version import get_version
19
+
20
+ SELF_PROFILE_ENDPOINT = "/aweme/v1/web/user/profile/self/"
21
+ SECONDS_PER_DAY = 86400
22
+ AGE_DAYS_PRECISION = 1
23
+ ONLINE_DETAIL_MAX_CHARS = 120
24
+ ONLINE_LOGGED_IN = "logged_in"
25
+ ONLINE_LOGIN_REQUIRED = "login_required"
26
+ ONLINE_EMPTY = "empty"
27
+ ONLINE_ERROR = "error"
28
+ ONLINE_SKIPPED = "skipped"
29
+ # verb -> 它额外依赖的本地组件(deps 里的键名)
30
+ VERB_DEPENDENCIES = {
31
+ "login": ("playwright",),
32
+ "live": ("ffmpeg",),
33
+ "transcribe": ("whisper",),
34
+ }
35
+ # 能力级别 -> (满足它的登录状态, 不满足时写进 blocked_by 的名字)
36
+ _LEVEL_REQUIREMENTS = {
37
+ "cookie": ((state.LOGIN_STATUS_DEVICE_ONLY, state.LOGIN_STATUS_LOGGED_IN), "cookies"),
38
+ "login": ((state.LOGIN_STATUS_LOGGED_IN,), "login"),
39
+ }
40
+
41
+
42
+ def register(subparsers: Any) -> None:
43
+ parser = subparsers.add_parser(
44
+ "doctor", help="report environment, login state and which verbs are usable"
45
+ )
46
+ parser.add_argument(
47
+ "--online",
48
+ action="store_true",
49
+ help="also verify the saved login with one request to Douyin",
50
+ )
51
+ parser.set_defaults(handler=run_doctor)
52
+
53
+
54
+ def _state_dir_info() -> Dict[str, Any]:
55
+ path = state.state_dir()
56
+ return {"path": str(path), "exists": path.is_dir(), "mode": state.mode_string(path)}
57
+
58
+
59
+ def _cookies_info(cookies: Dict[str, str]) -> Dict[str, Any]:
60
+ path = state.cookies_path()
61
+ exists = path.is_file()
62
+ mtime = None
63
+ age_days = None
64
+ if exists:
65
+ stamp = path.stat().st_mtime
66
+ mtime = datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).isoformat()
67
+ age_days = round(max(time.time() - stamp, 0) / SECONDS_PER_DAY, AGE_DAYS_PRECISION)
68
+ return {
69
+ "path": str(path),
70
+ "exists": exists,
71
+ "mode": state.mode_string(path),
72
+ "mtime": mtime,
73
+ "age_days": age_days,
74
+ "keys_present": state.keys_present(cookies),
75
+ "login_status": state.login_status(cookies),
76
+ }
77
+
78
+
79
+ def _module_dep(module: str, *pip_args: str, hint: str = "") -> Dict[str, Any]:
80
+ try:
81
+ available = importlib.util.find_spec(module) is not None
82
+ except (ImportError, ValueError):
83
+ available = False
84
+ info: Dict[str, Any] = {"available": available}
85
+ if not available:
86
+ info["install"] = install_action(*pip_args, hint=hint)
87
+ return info
88
+
89
+
90
+ def _ffmpeg_dep() -> Dict[str, Any]:
91
+ from core.ffmpeg import resolve_ffmpeg_path
92
+
93
+ path = resolve_ffmpeg_path()
94
+ info: Dict[str, Any] = {"available": bool(path), "path": path or None}
95
+ if not path:
96
+ info["install"] = install_action("imageio-ffmpeg==0.6.0")
97
+ return info
98
+
99
+
100
+ def _deps_info() -> Dict[str, Any]:
101
+ return {
102
+ "ffmpeg": _ffmpeg_dep(),
103
+ "playwright": _module_dep(
104
+ "playwright",
105
+ "playwright",
106
+ hint="Then run: %s" % python_command("-m", "playwright", "install", "chromium"),
107
+ ),
108
+ "whisper": _module_dep("whisper", "openai-whisper"),
109
+ }
110
+
111
+
112
+ def _verb_blockers(verb: str, login_state: str, deps: Dict[str, Any]) -> List[str]:
113
+ blockers = []
114
+ requirement = _LEVEL_REQUIREMENTS.get(VERB_CAPABILITIES[verb])
115
+ if requirement is not None and login_state not in requirement[0]:
116
+ blockers.append(requirement[1])
117
+ for dep in VERB_DEPENDENCIES.get(verb, ()):
118
+ if not deps[dep]["available"]:
119
+ blockers.append(dep)
120
+ return blockers
121
+
122
+
123
+ def _capabilities_info(login_state: str, deps: Dict[str, Any]) -> Dict[str, Any]:
124
+ verbs = {}
125
+ for verb, level in VERB_CAPABILITIES.items():
126
+ blockers = _verb_blockers(verb, login_state, deps)
127
+ verbs[verb] = {"level": level, "ready": not blockers, "blocked_by": blockers}
128
+ return {"verbs": verbs, "url_types": dict(URL_TYPE_CAPABILITIES)}
129
+
130
+
131
+ def _online_result(status: str, detail: str = "") -> Dict[str, Any]:
132
+ return {
133
+ "endpoint": SELF_PROFILE_ENDPOINT,
134
+ "status": status,
135
+ "logged_in": status == ONLINE_LOGGED_IN,
136
+ "detail": clean_text(detail, ONLINE_DETAIL_MAX_CHARS),
137
+ }
138
+
139
+
140
+ async def _online_check(cookies: Dict[str, str]) -> Dict[str, Any]:
141
+ """用 profile/self 验证登录态。没有会话 cookie 时不发请求。"""
142
+ if state.login_status(cookies) != state.LOGIN_STATUS_LOGGED_IN:
143
+ return _online_result(ONLINE_SKIPPED, "no session cookie saved; nothing to verify")
144
+
145
+ from core.api_client import DouyinAPIClient, LoginRequiredError
146
+
147
+ proxy = state.build_config().get("proxy")
148
+ try:
149
+ async with DouyinAPIClient(cookies, proxy=proxy) as client:
150
+ user = await client.get_self_info()
151
+ except LoginRequiredError:
152
+ return _online_result(ONLINE_LOGIN_REQUIRED, "Douyin rejected the saved session")
153
+ except Exception as exc: # noqa: BLE001 — doctor 只报告,不因联网失败而失败
154
+ return _online_result(ONLINE_ERROR, type(exc).__name__)
155
+ if user:
156
+ return _online_result(ONLINE_LOGGED_IN)
157
+ return _online_result(ONLINE_EMPTY, "empty response (rate limit, anti-bot or network)")
158
+
159
+
160
+ async def run_doctor(args: argparse.Namespace) -> Dict[str, Any]:
161
+ cookies = state.read_cookies()
162
+ for warning in state.permission_warnings():
163
+ add_warning(warning)
164
+ deps = _deps_info()
165
+ online: Optional[Dict[str, Any]] = None
166
+ if args.online:
167
+ online = await _online_check(cookies)
168
+ return {
169
+ "version": get_version(),
170
+ "python": platform.python_version(),
171
+ "platform": platform.platform(),
172
+ "executable": sys.executable,
173
+ "state_dir": _state_dir_info(),
174
+ "cookies": _cookies_info(cookies),
175
+ "deps": deps,
176
+ "capabilities": _capabilities_info(state.login_status(cookies), deps),
177
+ "online_check": online,
178
+ }
@@ -0,0 +1,84 @@
1
+ """能力表:哪些端点受门禁、每种 url_type 和每个 verb 需要什么级别。
2
+
3
+ `tests/test_agent_capabilities_drift.py` 会把这里的表与 `core/api_client.py`、
4
+ `utils/validators.py` 的实际情况对照,共享代码变了而这里没跟上时测试会失败。
5
+ """
6
+
7
+ ANONYMOUS = "anonymous"
8
+ COOKIE = "cookie"
9
+ LOGIN = "login"
10
+ LOCAL = "local"
11
+ ARGUS_GATED = "argus_gated"
12
+ UNSUPPORTED = "unsupported"
13
+ LEVELS = (ANONYMOUS, COOKIE, LOGIN, LOCAL, ARGUS_GATED, UNSUPPORTED)
14
+
15
+ # core/api_client.py 中经 `_request_json_gated` 发出的全部路径。
16
+ # 开源 CLI 没有 page bridge,这些请求会被 ArgusSecurityPlugin 确定性拒绝。
17
+ GATED_ENDPOINTS = frozenset(
18
+ {
19
+ "/aweme/v1/web/aweme/detail/",
20
+ "/aweme/v1/web/aweme/post/",
21
+ "/aweme/v1/web/aweme/favorite/",
22
+ "/aweme/v1/web/aweme/listcollection/",
23
+ "/aweme/v1/web/mix/list/",
24
+ "/aweme/v1/web/mix/detail/",
25
+ "/aweme/v1/web/mix/aweme/",
26
+ "/aweme/v1/web/mix/listcollection/",
27
+ "/aweme/v1/web/series/list/",
28
+ "/aweme/v1/web/music/list/",
29
+ "/aweme/v1/web/music/detail/",
30
+ "/aweme/v1/web/music/aweme/",
31
+ "/aweme/v1/web/collects/list/",
32
+ "/aweme/v1/web/collects/video/list/",
33
+ }
34
+ )
35
+
36
+ VERB_CAPABILITIES = {
37
+ "doctor": LOCAL,
38
+ "inspect": ANONYMOUS,
39
+ "hot": ANONYMOUS,
40
+ "search": LOGIN,
41
+ "user": COOKIE,
42
+ "login": LOCAL,
43
+ "live": ANONYMOUS,
44
+ "transcribe": LOCAL,
45
+ }
46
+
47
+ # `parse_url_type` 对未解析短链返回的占位类型;inspect 先解析短链,它不进能力表。
48
+ UNRESOLVED_URL_TYPE = "short"
49
+
50
+ # url_type(取值以 utils/validators.parse_url_type 为准)-> 能力级别
51
+ URL_TYPE_CAPABILITIES = {
52
+ "video": ARGUS_GATED,
53
+ "gallery": ARGUS_GATED,
54
+ "collection": ARGUS_GATED,
55
+ "music": ARGUS_GATED,
56
+ "user": COOKIE,
57
+ "live": ANONYMOUS,
58
+ "live_replay": UNSUPPORTED,
59
+ "lvdetail": UNSUPPORTED,
60
+ }
61
+
62
+ URL_TYPE_SUGGESTED_VERB = {
63
+ "user": "user",
64
+ "live": "live",
65
+ }
66
+
67
+ URL_TYPE_NOTES = {
68
+ "video": "Single-video download needs the Argus-gated aweme/detail endpoint.",
69
+ "gallery": "Image-post download needs the Argus-gated aweme/detail endpoint.",
70
+ "collection": "Collection (mix) download needs the Argus-gated mix endpoints.",
71
+ "music": "Music download needs the Argus-gated music endpoints.",
72
+ "user": (
73
+ "Profile info is available via `user`. Downloading this user's posts is "
74
+ "Argus-gated and not offered in agent mode."
75
+ ),
76
+ "live": "A live room can be recorded with `live <url> --max-duration S` while it is on air.",
77
+ "live_replay": "Live replays are not offered in agent mode v1.",
78
+ "lvdetail": "Douyin cinema (lvdetail) content is DRM-encrypted and cannot be downloaded.",
79
+ }
80
+ UNKNOWN_URL_NOTE = "Not a recognised Douyin URL."
81
+
82
+
83
+ def url_type_level(url_type: object) -> str:
84
+ return URL_TYPE_CAPABILITIES.get(str(url_type or ""), UNSUPPORTED)
@@ -0,0 +1,152 @@
1
+ """`hot`、`search`、`user`:联网只读 verb,只输出投影字段,绝不含媒体直链。接口见 entry.py。
2
+
3
+ `search` 的实现在 discovery_search.py,这里只负责注册。
4
+ """
5
+
6
+ import argparse
7
+ import re
8
+ from typing import Any, Dict
9
+ from urllib.parse import quote
10
+
11
+ from agent_mode import discovery_search, state
12
+ from agent_mode.capabilities import URL_TYPE_NOTES
13
+ from agent_mode.discovery_common import (
14
+ ID_MAX_CHARS,
15
+ RETRY_AFTER_S,
16
+ as_dict,
17
+ as_int,
18
+ open_client,
19
+ upstream_empty,
20
+ )
21
+ from agent_mode.entry import bounded_int
22
+ from agent_mode.envelope import DESC_MAX_CHARS, NAME_MAX_CHARS, AgentError, clean_text
23
+
24
+ HOT_DEFAULT_MAX = 50
25
+ HOT_MAX_LIMIT = 100
26
+ SEARCH_URL_PREFIX = "https://www.douyin.com/search/"
27
+ PUBLISH_DAYS_HELP = "only posts published within N days (0 = any time)"
28
+ # sec_uid 都以这个前缀开头;抖音号、数字 uid 等其他裸 id 不能拿去查主页
29
+ SEC_UID_PREFIX = "MS4wLjAB"
30
+ SEC_UID_HINT = "pass the profile URL or a sec_uid (starts with %s)" % SEC_UID_PREFIX
31
+ _SEC_UID_RE = re.compile(r"^[A-Za-z0-9_-]+$")
32
+ _HOT_OPTIONAL_FIELDS = ("label", "video_count")
33
+ _USER_STAT_FIELDS = ("follower_count", "following_count", "aweme_count", "total_favorited")
34
+
35
+
36
+ def register(subparsers: Any) -> None:
37
+ hot = subparsers.add_parser("hot", help="list the Douyin hot-search board")
38
+ hot.add_argument(
39
+ "--max", type=bounded_int(1, HOT_MAX_LIMIT), default=HOT_DEFAULT_MAX, metavar="N"
40
+ )
41
+ hot.set_defaults(handler=run_hot)
42
+ search = subparsers.add_parser("search", help="search posts by keyword (needs login)")
43
+ search.add_argument("keyword")
44
+ search.add_argument(
45
+ "--max",
46
+ type=bounded_int(1, discovery_search.SEARCH_MAX_LIMIT),
47
+ default=discovery_search.SEARCH_DEFAULT_MAX,
48
+ metavar="N",
49
+ )
50
+ search.add_argument(
51
+ "--sort", choices=sorted(discovery_search.SEARCH_SORT_TYPES), default="general"
52
+ )
53
+ search.add_argument(
54
+ "--publish-days",
55
+ type=int,
56
+ choices=discovery_search.SEARCH_PUBLISH_DAYS,
57
+ default=0,
58
+ help=PUBLISH_DAYS_HELP,
59
+ )
60
+ search.set_defaults(handler=discovery_search.run_search)
61
+ user = subparsers.add_parser("user", help="show a user's public profile (needs cookies)")
62
+ user.add_argument("target", help="profile URL or sec_uid")
63
+ user.set_defaults(handler=run_user)
64
+
65
+
66
+ def _project_hot_item(raw: Dict[str, Any], rank: int) -> Dict[str, Any]:
67
+ word = clean_text(raw.get("word"), NAME_MAX_CHARS)
68
+ item = {"rank": rank, "word": word, "hot_value": as_int(raw.get("hot_value"))}
69
+ for field in _HOT_OPTIONAL_FIELDS:
70
+ value = as_int(raw.get(field))
71
+ if value:
72
+ item[field] = value
73
+ item["search_url"] = SEARCH_URL_PREFIX + quote(word, safe="")
74
+ return item
75
+
76
+
77
+ async def run_hot(args: argparse.Namespace) -> Dict[str, Any]:
78
+ from core.api_client import LoginRequiredError
79
+
80
+ try:
81
+ async with open_client(state.read_cookies()) as client:
82
+ page = await client.get_hot_search_board()
83
+ except LoginRequiredError:
84
+ raise AgentError("LOGIN_REQUIRED", "Douyin asked for a login for the hot board.") from None
85
+ rows = [raw for raw in page.get("items") or [] if clean_text(as_dict(raw).get("word"))]
86
+ items = [_project_hot_item(raw, rank) for rank, raw in enumerate(rows[: args.max], 1)]
87
+ if not items:
88
+ empty = {"items": [], "count": 0}
89
+ raise upstream_empty("Douyin returned an empty hot board.", RETRY_AFTER_S, empty)
90
+ return {"items": items, "count": len(items)}
91
+
92
+
93
+ def _not_a_profile() -> AgentError:
94
+ action = {"kind": "fix_usage", "hint": SEC_UID_HINT}
95
+ return AgentError("USAGE", "Expected a Douyin profile URL or a sec_uid.", action=action)
96
+
97
+
98
+ async def _resolve_sec_uid(target: str) -> str:
99
+ """裸 sec_uid 直接用;否则按 URL(含短链、分享文案)解析出 sec_uid。"""
100
+ from agent_mode.inspect_cmd import resolve_input_url
101
+ from core.url_parser import URLParser
102
+
103
+ bare = target.strip()
104
+ if _SEC_UID_RE.match(bare):
105
+ if not bare.startswith(SEC_UID_PREFIX):
106
+ # 抖音号之类的裸 id 查不到主页,不发请求
107
+ raise _not_a_profile()
108
+ return bare
109
+ url = await resolve_input_url(target)
110
+ if url is None:
111
+ raise upstream_empty("Could not resolve the link.", RETRY_AFTER_S, {"sec_uid": None})
112
+ parsed = URLParser.parse(url) or {}
113
+ sec_uid = str(parsed.get("sec_uid") or "") if parsed.get("type") == "user" else ""
114
+ if not _SEC_UID_RE.match(sec_uid):
115
+ raise _not_a_profile()
116
+ return sec_uid
117
+
118
+
119
+ def _project_user(raw: Dict[str, Any], sec_uid: str) -> Dict[str, Any]:
120
+ data: Dict[str, Any] = {
121
+ "sec_uid": clean_text(raw.get("sec_uid") or sec_uid, ID_MAX_CHARS),
122
+ "uid": clean_text(raw.get("uid"), ID_MAX_CHARS),
123
+ "unique_id": clean_text(raw.get("unique_id") or raw.get("short_id"), NAME_MAX_CHARS),
124
+ "nickname": clean_text(raw.get("nickname"), NAME_MAX_CHARS),
125
+ "signature": clean_text(raw.get("signature"), DESC_MAX_CHARS),
126
+ }
127
+ for field in _USER_STAT_FIELDS:
128
+ data[field] = as_int(raw.get(field))
129
+ ip_location = clean_text(raw.get("ip_location"), NAME_MAX_CHARS)
130
+ if ip_location:
131
+ data["ip_location"] = ip_location
132
+ data["note"] = URL_TYPE_NOTES["user"]
133
+ return data
134
+
135
+
136
+ async def run_user(args: argparse.Namespace) -> Dict[str, Any]:
137
+ cookies = state.read_cookies()
138
+ if not cookies:
139
+ # 没有 cookie 文件时 profile 接口拿不到数据,不发任何请求(含短链解析)
140
+ raise AgentError("LOGIN_REQUIRED", "The user verb needs saved cookies; none are saved.")
141
+
142
+ from core.api_client import LoginRequiredError
143
+
144
+ sec_uid = await _resolve_sec_uid(args.target)
145
+ try:
146
+ async with open_client(cookies) as client:
147
+ raw = await client.get_user_info(sec_uid)
148
+ except LoginRequiredError:
149
+ raise AgentError("LOGIN_REQUIRED", "Douyin asked for a login for this profile.") from None
150
+ if not isinstance(raw, dict) or not raw:
151
+ raise upstream_empty("Douyin returned no profile.", RETRY_AFTER_S, {"sec_uid": sec_uid})
152
+ return _project_user(raw, sec_uid)
@@ -0,0 +1,33 @@
1
+ """`hot`、`search`、`user` 共用的常量与小工具。"""
2
+
3
+ from typing import Any, Dict, Optional
4
+
5
+ from agent_mode import state
6
+ from agent_mode.envelope import AgentError, retry_action
7
+
8
+ RETRY_AFTER_S = 60
9
+ ID_MAX_CHARS = 128
10
+ EMPTY_HINT = "Likely a rate limit or anti-bot response. Wait before retrying; do not loop."
11
+
12
+
13
+ def as_int(value: Any) -> Optional[int]:
14
+ """平台数值可能是 int、数字字符串或缺失;解析不了就是 None。"""
15
+ try:
16
+ return None if isinstance(value, bool) else int(value)
17
+ except (TypeError, ValueError):
18
+ return None
19
+
20
+
21
+ def as_dict(value: Any) -> Dict[str, Any]:
22
+ return value if isinstance(value, dict) else {}
23
+
24
+
25
+ def open_client(cookies: Dict[str, str]) -> Any:
26
+ from core.api_client import DouyinAPIClient
27
+
28
+ return DouyinAPIClient(cookies, proxy=state.build_config().get("proxy"))
29
+
30
+
31
+ def upstream_empty(message: str, retry_after_s: int, data: Dict[str, Any]) -> AgentError:
32
+ action = retry_action(retry_after_s, EMPTY_HINT)
33
+ return AgentError("UPSTREAM_EMPTY", message, action=action, data=data)
@@ -0,0 +1,182 @@
1
+ """`search` 的实现:翻页、投影、区分「确实没有结果」和「请求失败或被拦」。"""
2
+
3
+ import argparse
4
+ import asyncio
5
+ import datetime
6
+ from typing import Any, Dict, List, Optional
7
+
8
+ from agent_mode import state
9
+ from agent_mode.discovery_common import (
10
+ EMPTY_HINT,
11
+ ID_MAX_CHARS,
12
+ as_dict,
13
+ as_int,
14
+ open_client,
15
+ upstream_empty,
16
+ )
17
+ from agent_mode.envelope import DESC_MAX_CHARS, NAME_MAX_CHARS, AgentError, clean_text, retry_action
18
+
19
+ SEARCH_DEFAULT_MAX = 20
20
+ SEARCH_MAX_LIMIT = 50
21
+ # 与 DouyinAPIClient.search_aweme 的 sort_type / publish_time 取值对应
22
+ SEARCH_SORT_TYPES = {"general": 0, "likes": 1, "latest": 2}
23
+ SEARCH_PUBLISH_DAYS = (0, 1, 7, 182)
24
+ SEARCH_PAGE_SIZE = 10
25
+ # 翻页间隔比人用 CLI 保守,避免 agent 连续翻页触发风控
26
+ SEARCH_PAGE_INTERVAL_S = 1.0
27
+ SEARCH_RETRY_AFTER_S = 120
28
+ KEYWORD_MAX_CHARS = 100
29
+ MS_PER_SECOND = 1000
30
+ DURATION_PRECISION = 1
31
+ ISO_UTC_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
32
+ UPSTREAM_OK_STATUS = 0
33
+ # 状态码为 0 但其实被风控拦下时,响应里 search_nil_info.search_nil_type 的取值
34
+ BLOCKED_NIL_TYPES = ("verify_check", "hit_shark")
35
+ # 与 core/downloader_base.py 的 BaseDownloader._GALLERY_AWEME_TYPES 一致(有漂移测试)
36
+ GALLERY_AWEME_TYPES = (2, 68, 150)
37
+ # media_type -> 分享链接的路径段
38
+ SHARE_URL_TEMPLATE = "https://www.douyin.com/%s/%s"
39
+ SHARE_URL_SEGMENTS = {"video": "video", "gallery": "note"}
40
+ _STAT_FIELDS = ("digg", "comment", "share", "collect")
41
+
42
+
43
+ def _media_type(raw: Dict[str, Any]) -> str:
44
+ image_post = as_dict(raw.get("image_post_info"))
45
+ has_images = raw.get("images") or image_post.get("images") or image_post.get("image_list")
46
+ return "gallery" if has_images or raw.get("aweme_type") in GALLERY_AWEME_TYPES else "video"
47
+
48
+
49
+ def _iso_utc(epoch: Any) -> Optional[str]:
50
+ seconds = as_int(epoch) or 0
51
+ try:
52
+ moment = datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc)
53
+ except (OverflowError, OSError, ValueError):
54
+ return None
55
+ return moment.strftime(ISO_UTC_FORMAT) if seconds > 0 else None
56
+
57
+
58
+ def _duration_s(raw: Dict[str, Any]) -> Optional[float]:
59
+ millis = as_int(as_dict(raw.get("video")).get("duration")) or as_int(raw.get("duration"))
60
+ return round(millis / MS_PER_SECOND, DURATION_PRECISION) if millis and millis > 0 else None
61
+
62
+
63
+ def _project_aweme(raw: Dict[str, Any]) -> Optional[Dict[str, Any]]:
64
+ """只挑契约里的字段;video / images 等带直链的子树整个不输出。"""
65
+ aweme_id = clean_text(raw.get("aweme_id"), ID_MAX_CHARS)
66
+ if not aweme_id.isdigit():
67
+ return None
68
+ author = as_dict(raw.get("author"))
69
+ stats = as_dict(raw.get("statistics"))
70
+ media_type = _media_type(raw)
71
+ return {
72
+ "aweme_id": aweme_id,
73
+ "desc": clean_text(raw.get("desc"), DESC_MAX_CHARS),
74
+ "author": {
75
+ "nickname": clean_text(author.get("nickname"), NAME_MAX_CHARS),
76
+ "sec_uid": clean_text(author.get("sec_uid"), ID_MAX_CHARS),
77
+ },
78
+ "stats": {name: as_int(stats.get(name + "_count")) for name in _STAT_FIELDS},
79
+ "create_time": _iso_utc(raw.get("create_time")),
80
+ "duration_s": _duration_s(raw),
81
+ "media_type": media_type,
82
+ "share_url": SHARE_URL_TEMPLATE % (SHARE_URL_SEGMENTS[media_type], aweme_id),
83
+ }
84
+
85
+
86
+ def _absorb_page(page: Dict[str, Any], items: List[Dict[str, Any]], limit: int) -> bool:
87
+ """把一页结果去重后并入 items;返回这一页是否还有没放进去的新条目。"""
88
+ seen = {item["aweme_id"] for item in items}
89
+ for raw in page.get("items") or []:
90
+ projected = _project_aweme(as_dict(raw))
91
+ if projected is None or projected["aweme_id"] in seen:
92
+ continue
93
+ if len(items) >= limit:
94
+ return True
95
+ seen.add(projected["aweme_id"])
96
+ items.append(projected)
97
+ return False
98
+
99
+
100
+ class _PageFailed(Exception):
101
+ """某一页请求失败或被拦;调用方按已拿到的条数决定错误码。"""
102
+
103
+
104
+ def _is_failed_page(page: Dict[str, Any]) -> bool:
105
+ """空页是请求失败或被拦,而不是确实没有结果(或自然翻到底)。
106
+
107
+ 只有上游明确报成功(响应体里 status_code 为 0)且没有风控标记,才算真的没有结果。
108
+ """
109
+ if page.get("items"):
110
+ return False
111
+ raw = as_dict(page.get("raw"))
112
+ if page.get("status_code") or as_int(raw.get("status_code")) != UPSTREAM_OK_STATUS:
113
+ return True
114
+ nil_type = as_dict(raw.get("search_nil_info")).get("search_nil_type")
115
+ return nil_type in BLOCKED_NIL_TYPES
116
+
117
+
118
+ async def _collect_search(client: Any, args: argparse.Namespace, items: List[Any]) -> bool:
119
+ """翻页直到凑够 args.max 或到底,结果就地写进 items;返回 has_more。"""
120
+ offset = 0
121
+ while True:
122
+ page = await client.search_aweme(
123
+ args.keyword,
124
+ offset=offset,
125
+ count=SEARCH_PAGE_SIZE,
126
+ sort_type=SEARCH_SORT_TYPES[args.sort],
127
+ publish_time=args.publish_days,
128
+ )
129
+ if _is_failed_page(page):
130
+ raise _PageFailed("empty response at offset %d" % offset)
131
+ leftover = _absorb_page(page, items, args.max)
132
+ has_more = bool(page.get("has_more"))
133
+ if len(items) >= args.max:
134
+ return leftover or has_more
135
+ next_offset = as_int(page.get("max_cursor")) or 0
136
+ if not has_more or not page.get("items") or next_offset == offset:
137
+ return False
138
+ offset = next_offset
139
+ await asyncio.sleep(SEARCH_PAGE_INTERVAL_S)
140
+
141
+
142
+ def _search_data(keyword: str, items: List[Dict[str, Any]], has_more: bool) -> Dict[str, Any]:
143
+ keyword = clean_text(keyword, KEYWORD_MAX_CHARS)
144
+ return {"keyword": keyword, "items": items, "count": len(items), "has_more": has_more}
145
+
146
+
147
+ def _search_failure(args: argparse.Namespace, items: List[Any], reason: str) -> AgentError:
148
+ if not items:
149
+ empty = _search_data(args.keyword, [], False)
150
+ return upstream_empty("Search returned no items.", SEARCH_RETRY_AFTER_S, empty)
151
+ return AgentError(
152
+ "PARTIAL",
153
+ "Search stopped after %d of %d requested items (%s)." % (len(items), args.max, reason),
154
+ action=retry_action(SEARCH_RETRY_AFTER_S, EMPTY_HINT),
155
+ data=_search_data(args.keyword, items, True),
156
+ )
157
+
158
+
159
+ async def run_search(args: argparse.Namespace) -> Dict[str, Any]:
160
+ cookies = state.read_cookies()
161
+ if state.login_status(cookies) != state.LOGIN_STATUS_LOGGED_IN:
162
+ # 没有会话 cookie 时搜索必然被拒,不构造 client,也不发请求
163
+ raise AgentError("LOGIN_REQUIRED", "Search needs a logged-in session; none is saved.")
164
+
165
+ from core.api_client import LoginRequiredError
166
+
167
+ items: List[Dict[str, Any]] = []
168
+ try:
169
+ async with open_client(cookies) as client:
170
+ has_more = await _collect_search(client, args, items)
171
+ except LoginRequiredError:
172
+ partial = _search_data(args.keyword, items, True) if items else None
173
+ message = "Douyin rejected the saved session for search."
174
+ raise AgentError("LOGIN_REQUIRED", message, data=partial) from None
175
+ except _PageFailed as exc:
176
+ raise _search_failure(args, items, str(exc)) from None
177
+ except Exception as exc: # noqa: BLE001 — 已有部分结果时带着返回,否则交给入口报 INTERNAL
178
+ if not items:
179
+ raise
180
+ raise _search_failure(args, items, type(exc).__name__) from None
181
+ # 走到这里而 items 为空:上游明确成功,只是这个关键词没有结果
182
+ return _search_data(args.keyword, items, has_more)