douyin-cli 5.3.260515__py3-none-any.whl

backend/__init__.py

@@ -0,0 +1 @@
+"""DouyinCrawler backend package."""

backend/cli.py

@@ -0,0 +1,491 @@
+"""Command line interface for DouyinCrawler."""
+
+from __future__ import annotations
+
+import traceback
+from collections.abc import Iterator
+from dataclasses import dataclass
+from json import JSONDecodeError
+from pathlib import Path
+
+import click
+import ujson as json
+from loguru import logger
+
+from backend.constants import APP_VERSION, SETTINGS_FILE
+from backend.lib.cookies import CookieManager
+from backend.lib.douyin import Douyin
+from backend.lib.douyin.openapi import DouyinOpenAPIClient, DouyinOpenAPIError
+from backend.lib.download import download
+from backend.settings import settings
+
+ACCOUNT_ONLY_TYPES = {"favorite", "collection", "following", "follower"}
+NO_DOWNLOAD_TYPES = {"following", "follower"}
+
+BANNER = rf"""
+  ____                    _          ____                    _
+ |  _ \  ___  _   _ _   _(_)_ __    / ___|_ __ __ ___      _| | ___ _ __
+ | | | |/ _ \| | | | | | | | '_ \  | |   | '__/ _` \ \ /\ / / |/ _ \ '__|
+ | |_| | (_) | |_| | |_| | | | | | | |___| | | (_| |\ V  V /| |  __/ |
+ |____/ \___/ \__,_|\__, |_|_| |_|  \____|_|  \__,_| \_/\_/ |_|\___|_|
+                    |___/
+                              V{APP_VERSION}
+                Github: https://github.com/erma0/douyin
+"""
+
+
+@dataclass(frozen=True)
+class CrawlOptions:
+    """Normalized options used by one or more crawl tasks."""
+
+    limit: int
+    no_download: bool
+    crawl_type: str
+    output_path: str
+    cookie: str
+    filters: dict[str, str]
+    download_title: bool
+    download_cover: bool
+
+
+@click.group(invoke_without_command=True)
+@click.pass_context
+@click.option(
+    "-u",
+    "--urls",
+    type=click.STRING,
+    multiple=True,
+    help="作品/账号/话题/音乐等类型的URL链接/ID或搜索关键词，也可输入文件路径（文件内一行一个），可多次输入。",
+)
+@click.option(
+    "-l",
+    "--limit",
+    type=click.INT,
+    default=0,
+    help="限制最大采集数量，默认不限制（0表示不限制）",
+)
+@click.option(
+    "--no-download",
+    is_flag=True,
+    help="不下载文件，仅采集数据",
+)
+@click.option(
+    "-t",
+    "--type",
+    "crawl_type",
+    type=click.Choice(
+        [
+            "post",
+            "favorite",
+            "music",
+            "hashtag",
+            "search",
+            "following",
+            "follower",
+            "collection",
+            "mix",
+            "aweme",
+        ],
+        case_sensitive=False,
+    ),
+    default="post",
+    help="采集类型，默认为post（主页作品）。支持：post/favorite/music/hashtag/search/following/follower/collection/mix/aweme",
+)
+@click.option(
+    "-p",
+    "--path",
+    "output_path",
+    type=click.STRING,
+    default="下载",
+    help="下载文件夹路径，默认为[下载]",
+)
+@click.option(
+    "-c",
+    "--cookie",
+    type=click.STRING,
+    help=f"已登录账号的cookie，可填写在 {SETTINGS_FILE} 中",
+)
+@click.option(
+    "--sort-type",
+    type=click.Choice(["0", "1", "2"], case_sensitive=False),
+    help="搜索排序（仅search类型）：0=综合，1=最多点赞，2=最新",
+)
+@click.option(
+    "--publish-time",
+    type=click.Choice(["0", "1", "7", "180"], case_sensitive=False),
+    help="发布时间（仅search类型）：0=不限，1=一天内，7=一周内，180=半年内",
+)
+@click.option(
+    "--filter-duration",
+    type=click.Choice(["", "0-1", "1-5", "5-10000"], case_sensitive=False),
+    help="视频时长（仅search类型）：空=不限，0-1=1分钟以下，1-5=1-5分钟，5-10000=5分钟以上",
+)
+@click.option(
+    "--download-title",
+    is_flag=True,
+    help="下载标题文本文件",
+)
+@click.option(
+    "--download-cover",
+    is_flag=True,
+    help="下载封面图片",
+)
+def main(
+    ctx: click.Context,
+    urls: tuple[str, ...],
+    limit: int,
+    no_download: bool,
+    crawl_type: str,
+    output_path: str,
+    cookie: str | None,
+    sort_type: str | None,
+    publish_time: str | None,
+    filter_duration: str | None,
+    download_title: bool,
+    download_cover: bool,
+) -> None:
+    """抖音数据采集命令行工具."""
+    if ctx.invoked_subcommand is not None:
+        return
+
+    click.echo(BANNER)
+
+    cookie_value = load_cookie(cookie)
+    if not cookie_value or not validate_cookie(cookie_value):
+        return
+
+    targets = resolve_targets(urls, crawl_type)
+    if targets is None:
+        return
+
+    options = CrawlOptions(
+        limit=limit,
+        no_download=no_download,
+        crawl_type=crawl_type,
+        output_path=output_path,
+        cookie=cookie_value,
+        filters=build_filters(sort_type, publish_time, filter_duration),
+        download_title=download_title,
+        download_cover=download_cover,
+    )
+    run_targets(targets, options)
+
+
+@main.group()
+def api() -> None:
+    """调用抖音开放平台官方 OpenAPI."""
+
+
+@api.command("client-token")
+@click.option("--client-key", envvar="DOUYIN_CLIENT_KEY", required=True)
+@click.option("--client-secret", envvar="DOUYIN_CLIENT_SECRET", required=True)
+def api_client_token(client_key: str, client_secret: str) -> None:
+    """获取 client_token."""
+    with DouyinOpenAPIClient() as client:
+        echo_json(client.client_token(client_key, client_secret))
+
+
+@api.command("access-token")
+@click.option("--client-key", envvar="DOUYIN_CLIENT_KEY", required=True)
+@click.option("--client-secret", envvar="DOUYIN_CLIENT_SECRET", required=True)
+@click.option("--code", required=True, help="OAuth 授权码")
+def api_access_token(client_key: str, client_secret: str, code: str) -> None:
+    """用 OAuth code 换取 access_token."""
+    with DouyinOpenAPIClient() as client:
+        echo_json(client.access_token(client_key, client_secret, code))
+
+
+@api.command("request")
+@click.argument("method")
+@click.argument("path")
+@click.option("--token", envvar="DOUYIN_ACCESS_TOKEN", help="access_token/client_token")
+@click.option("--param", "params", multiple=True, help="查询参数，格式 key=value")
+@click.option("--json", "json_text", help="JSON 请求体")
+@click.option("--form", "forms", multiple=True, help="表单参数，格式 key=value")
+@click.option("--header", "headers", multiple=True, help="额外请求头，格式 key=value")
+def api_request(
+    method: str,
+    path: str,
+    token: str | None,
+    params: tuple[str, ...],
+    json_text: str | None,
+    forms: tuple[str, ...],
+    headers: tuple[str, ...],
+) -> None:
+    """调用任意官方 OpenAPI 路径."""
+    try:
+        json_body = parse_json_body(json_text)
+        with DouyinOpenAPIClient() as client:
+            echo_json(
+                client.request(
+                    method,
+                    path,
+                    token=token,
+                    params=parse_key_values(params),
+                    json_body=json_body,
+                    form=parse_key_values(forms),
+                    headers=parse_key_values(headers),
+                ),
+            )
+    except (DouyinOpenAPIError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+
+
+def build_filters(
+    sort_type: str | None,
+    publish_time: str | None,
+    filter_duration: str | None,
+) -> dict[str, str]:
+    """Build search filter arguments."""
+    filters = {}
+    if sort_type:
+        filters["sort_type"] = sort_type
+    if publish_time:
+        filters["publish_time"] = publish_time
+    if filter_duration is not None:
+        filters["filter_duration"] = filter_duration
+    return filters
+
+
+def echo_json(data: dict) -> None:
+    """Print a JSON response."""
+    click.echo(json.dumps(data, ensure_ascii=False, indent=2))
+
+
+def parse_json_body(text: str | None) -> dict | list | None:
+    """Parse a JSON body option."""
+    if text is None:
+        return None
+    try:
+        data = json.loads(text)
+    except JSONDecodeError as exc:
+        msg = f"--json 不是合法 JSON: {exc}"
+        raise ValueError(msg) from exc
+    if not isinstance(data, (dict, list)):
+        msg = "--json 必须是 JSON object 或 array"
+        raise ValueError(msg)
+    return data
+
+
+def parse_key_values(values: tuple[str, ...]) -> dict[str, str] | None:
+    """Parse repeated key=value CLI options."""
+    if not values:
+        return None
+    parsed = {}
+    for value in values:
+        if "=" not in value:
+            msg = f"参数必须是 key=value 格式: {value}"
+            raise ValueError(msg)
+        key, item_value = value.split("=", 1)
+        if not key:
+            msg = f"参数 key 不能为空: {value}"
+            raise ValueError(msg)
+        parsed[key] = item_value
+    return parsed
+
+
+def load_cookie(cookie: str | None) -> str | None:
+    """Load cookie from CLI, config, or interactive prompt."""
+    if cookie is not None:
+        logger.info("正在加载命令行指定的Cookie...")
+        cookie_value = cookie.strip()
+        if cookie_value:
+            return cookie_value
+        logger.error("无法加载指定的Cookie")
+        return None
+
+    cookie_value = settings.get("cookie", "").strip()
+    if cookie_value:
+        logger.info("✓ 已从配置文件加载Cookie")
+        return cookie_value
+
+    return prompt_for_cookie()
+
+
+def prompt_for_cookie() -> str | None:
+    """Prompt for cookie when no configured cookie exists."""
+    logger.warning("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+    logger.warning("⚠ 未找到Cookie配置")
+    logger.info("配置方法：")
+    logger.info(f"  方法1：在 {SETTINGS_FILE} 中设置 cookie 字段")
+    logger.info("  方法2：使用 -c 参数：douyin -c 'your_cookie'")
+    logger.warning("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+
+    try:
+        cookie_value = click.prompt(
+            "请粘贴Cookie字符串",
+            default="",
+            show_default=False,
+        )
+    except (click.Abort, EOFError):
+        logger.warning("\n用户取消输入")
+        return None
+
+    cookie_value = cookie_value.strip()
+    if not cookie_value:
+        logger.error("未输入Cookie，程序退出")
+        return None
+    logger.success("✓ Cookie已输入")
+    return cookie_value
+
+
+def validate_cookie(cookie: str) -> bool:
+    """Validate cookie before crawling."""
+    if CookieManager.validate_cookie(cookie):
+        logger.success("✓ Cookie验证通过")
+        return True
+
+    logger.error("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+    logger.error("✗ Cookie验证失败")
+    logger.info("可能原因：")
+    logger.info("  1. Cookie已过期，请重新获取")
+    logger.info("  2. Cookie格式不正确")
+    logger.info("  3. 账号已退出登录")
+    logger.error("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+    return False
+
+
+def resolve_targets(urls: tuple[str, ...], crawl_type: str) -> tuple[str, ...] | None:
+    """Resolve CLI targets or prompt for one when needed."""
+    if urls:
+        return urls
+
+    if crawl_type in ACCOUNT_ONLY_TYPES:
+        logger.info(f"采集本账号的 {crawl_type} 数据")
+        return ("",)
+
+    url_input = click.prompt(
+        f"采集类型：{crawl_type}，请输入目标关键词/URL链接/ID或文件路径",
+        default="",
+        show_default=False,
+    ).strip()
+    if url_input:
+        return (url_input,)
+
+    logger.error("未输入目标，退出程序")
+    return None
+
+
+def run_targets(targets: tuple[str, ...], options: CrawlOptions) -> None:
+    """Run all targets and print a summary."""
+    success_count = 0
+    fail_count = 0
+
+    for target in iter_targets(targets):
+        if run_task(target, options):
+            success_count += 1
+        else:
+            fail_count += 1
+
+    logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+    logger.success(f"✓ 任务完成：成功 {success_count} 个，失败 {fail_count} 个")
+    logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+
+
+def iter_targets(targets: tuple[str, ...]) -> Iterator[str | None]:
+    """Yield individual targets, expanding files when an argument is a path."""
+    for raw_target in targets:
+        target = raw_target.strip()
+        if not target:
+            continue
+
+        path = Path(target)
+        if not path.exists():
+            yield target
+            continue
+
+        logger.info(f"从文件读取目标：{target}")
+        lines = read_target_file(path)
+        if not lines:
+            logger.error(f"文件 [{target}] 中没有发现目标URL")
+            yield None
+            continue
+
+        logger.info(f"文件中共有 {len(lines)} 个目标")
+        for index, line in enumerate(lines, 1):
+            logger.info(f"处理第 {index}/{len(lines)} 个目标")
+            yield line
+
+
+def read_target_file(path: Path) -> list[str]:
+    """Read one target per non-empty line."""
+    try:
+        return [
+            line.strip()
+            for line in path.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        ]
+    except OSError as exc:
+        logger.error(f"读取文件失败: {exc}")
+        return []
+
+
+def run_task(target: str | None, options: CrawlOptions) -> bool:
+    """Run one crawl task."""
+    if target is None:
+        return False
+
+    try:
+        log_task_start(target, options)
+        douyin = create_crawler(target, options)
+        douyin.run()
+        maybe_download(douyin, options)
+        return True
+    except KeyboardInterrupt:
+        logger.warning("用户中断任务")
+        return False
+    except Exception as exc:
+        logger.error(f"任务执行失败: {exc}")
+        logger.debug(traceback.format_exc())
+        return False
+
+
+def log_task_start(target: str, options: CrawlOptions) -> None:
+    """Log the task configuration."""
+    logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+    logger.info("开始采集任务")
+    logger.info(f"  目标: {target or '本账号'}")
+    logger.info(f"  类型: {options.crawl_type}")
+    logger.info(f"  数量限制: {'不限' if options.limit == 0 else f'{options.limit}条'}")
+    if options.filters:
+        logger.info(f"  筛选条件: {options.filters}")
+    if options.download_title:
+        logger.info("  下载标题: ✓ 是")
+    if options.download_cover:
+        logger.info("  下载封面: ✓ 是")
+    logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
+
+
+def create_crawler(target: str, options: CrawlOptions) -> Douyin:
+    """Create the crawler for one target."""
+    return Douyin(
+        target=target,
+        limit=options.limit,
+        type=options.crawl_type,
+        down_path=options.output_path,
+        cookie=options.cookie,
+        user_agent=settings.get("userAgent", ""),
+        filters=options.filters,
+        enable_download_title=options.download_title
+        or settings.get("enableDownloadTitle", False),
+        enable_download_cover=options.download_cover
+        or settings.get("enableDownloadCover", False),
+    )
+
+
+def maybe_download(douyin: Douyin, options: CrawlOptions) -> None:
+    """Download crawled files when requested and supported."""
+    if options.no_download:
+        logger.info("已跳过下载（--no-download）")
+        return
+
+    if douyin.type in NO_DOWNLOAD_TYPES:
+        logger.info("此类型不需要下载文件")
+        return
+
+    logger.info("开始下载文件...")
+    download(douyin.down_path, douyin.aria2_conf)
+
+
+if __name__ == "__main__":
+    main()

backend/constants.py

@@ -0,0 +1,38 @@
+"""Application constants."""
+
+from pathlib import Path
+
+from loguru import logger
+
+# 兼容独立脚本运行和模块导入两种方式
+try:
+    from .utils.paths import get_app_root
+except ImportError:
+    from utils.paths import get_app_root
+
+try:
+    from importlib.metadata import version as get_version
+
+    APP_VERSION = get_version("douyin-cli")
+except Exception as e:
+    logger.debug(f"获取版本号失败: {e}")
+    APP_VERSION = "0.0.0"
+
+# 项目根目录（应用目录）
+PROJECT_ROOT = get_app_root()
+# 完整路径
+CONFIG_DIR = str(Path(PROJECT_ROOT) / "config")
+DOWNLOAD_DIR = str(Path(PROJECT_ROOT) / "download")
+SETTINGS_FILE = str(Path(CONFIG_DIR) / "settings.json")
+
+# 默认设置（用于首次运行创建配置文件）
+DEFAULT_SETTINGS = {
+    "cookie": "",
+    "userAgent": "",  # 留空则使用内置默认值 (RequestHeaders.USER_AGENT)
+    "downloadPath": DOWNLOAD_DIR,
+    "enableIncrementalFetch": True,
+    "enableDownloadTitle": False,
+    "enableDownloadCover": False,
+    "filenameFields": ["id", "title"],
+    "filenameSeparator": "_",
+}

backend/lib/__init__.py

@@ -0,0 +1,27 @@
+"""业务逻辑模块
+
+包含应用的核心业务逻辑和通用服务。
+
+模块结构：
+- douyin/: 抖音爬虫模块（核心业务）
+  - crawler.py: 爬虫主类，协调各模块完成数据采集
+  - client.py: API客户端，封装抖音API调用
+  - parser.py: 数据解析器，解析API返回的数据
+  - target.py: 目标处理器，识别和解析用户输入
+  - types.py: 类型定义和常量（作品类型、API端点、请求参数等）
+  - request.py: HTTP请求封装，处理签名和Cookie
+  - js/: JavaScript脚本（签名生成）
+- cookies.py: Cookie管理和验证
+- download.py: 调用系统 aria2c 下载文件
+
+使用示例：
+    from backend.lib.douyin import Douyin
+
+    douyin = Douyin(target="https://www.douyin.com/user/xxx", limit=10)
+    douyin.run()
+"""
+
+# 向后兼容：保持原有的导入方式
+from .douyin import Douyin
+
+__all__ = ["Douyin"]