parsehub 1.0.0__py3-none-any.whl

parsehub/__init__.py

@@ -0,0 +1,7 @@
+from .deps.submodule_manager import SubmoduleManager
+from .main import ParseHub
+
+manager = SubmoduleManager()
+manager.setup_all()
+
+__all__ = ["ParseHub"]

parsehub/config/__init__.py

@@ -0,0 +1,3 @@
+from .config import ParseHubConfig
+
+__all__ = ["ParseHubConfig"]

parsehub/config/config.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+import sys
+from os import getenv
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+
+
+@dataclass
+class ParseHubConfig:
+    DOWNLOAD_DIR = getenv("DOWNLOAD_DIR", Path(sys.argv[0]).parent / Path("downloads/"))
+    """默认下载目录"""
+
+    douyin_api = getenv("DOUYIN_API", "https://douyin.wtf")
+    """抖音解析API地址, 项目地址: https://github.com/Evil0ctal/Douyin_TikTok_Download_API"""
+
+    provider = getenv("PROVIDER", "openai").lower()
+    """模型提供商"""
+
+    api_key = getenv("API_KEY")
+    """API Key"""
+
+    base_url = getenv("BASE_URL", "https://api.openai.com/v1")
+    """API 地址"""
+
+    model = getenv("MODEL", "gpt-4o-mini")
+    """AI总结模型名称"""

parsehub/deps/submodule_manager.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+import sys
+from typing import Dict, Optional
+
+
+class SubmoduleManager:
+    def __init__(self):
+        self.deps_dir = Path(__file__).parent
+        self.path_mappings: Dict[str, Path] = {}
+
+    def add_submodule(
+        self, name: str, relative_path: str, source_dir: Optional[str] = None
+    ):
+        """添加子模块路径映射"""
+        submodule_path = self.deps_dir / relative_path
+        self.path_mappings[name] = submodule_path
+
+        # 添加主模块路径
+        if str(submodule_path) not in sys.path:
+            sys.path.insert(0, str(submodule_path))
+
+        # 添加源码目录路径（如果指定）
+        if source_dir:
+            source_path = submodule_path / source_dir
+            if str(source_path) not in sys.path:
+                sys.path.insert(0, str(source_path))
+
+    def setup_all(self):
+        """设置所有子模块的路径"""
+        # 添加所有需要的子模块
+        self.add_submodule("xhs", "xhs", "xhs")

parsehub/deps/xhs/locale/po_to_mo.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from subprocess import run
+
+__all__ = []
+ROOT = Path(__file__).resolve().parent
+
+
+def scan_directory():
+    return [
+        item.joinpath("LC_MESSAGES/xhs.po") for item in ROOT.iterdir() if item.is_dir()
+    ]
+
+
+def generate_map(files: list[Path]):
+    return [(i, i.with_suffix(".mo")) for i in files]
+
+
+def generate_mo(maps: list[tuple[Path, Path]]):
+    for i, j in maps:
+        command = f'msgfmt "{i}" -o "{j}"'
+        print(run(command, shell=True, text=True))
+
+
+if __name__ == "__main__":
+    generate_mo(generate_map(scan_directory()))

parsehub/deps/xhs/main.py

@@ -0,0 +1,94 @@
+from asyncio import run
+from asyncio.exceptions import CancelledError
+from contextlib import suppress
+from sys import argv
+
+from source import Settings
+from source import XHS
+from source import XHSDownloader
+from source import cli
+
+
+async def example():
+    """通过代码设置参数，适合二次开发"""
+    # 示例链接
+    error_link = "https://github.com/JoeanAmier/XHS_Downloader"
+    demo_link = "https://www.xiaohongshu.com/explore/xxxxxxxxxx"
+    multiple_links = f"{demo_link} {demo_link} {demo_link}"
+    # 实例对象
+    work_path = "D:\\"  # 作品数据/文件保存根路径，默认值：项目根路径
+    folder_name = "Download"  # 作品文件储存文件夹名称（自动创建），默认值：Download
+    name_format = "作品标题 作品描述"
+    user_agent = ""  # User-Agent
+    cookie = ""  # 小红书网页版 Cookie，无需登录，可选参数，登录状态对数据采集有影响
+    proxy = None  # 网络代理
+    timeout = 5  # 请求数据超时限制，单位：秒，默认值：10
+    chunk = 1024 * 1024 * 10  # 下载文件时，每次从服务器获取的数据块大小，单位：字节
+    max_retry = 2  # 请求数据失败时，重试的最大次数，单位：秒，默认值：5
+    record_data = False  # 是否保存作品数据至文件
+    image_format = "WEBP"  # 图文作品文件下载格式，支持：PNG、WEBP
+    folder_mode = False  # 是否将每个作品的文件储存至单独的文件夹
+    # async with XHS() as xhs:
+    #     pass  # 使用默认参数
+    async with XHS(
+        work_path=work_path,
+        folder_name=folder_name,
+        name_format=name_format,
+        user_agent=user_agent,
+        cookie=cookie,
+        proxy=proxy,
+        timeout=timeout,
+        chunk=chunk,
+        max_retry=max_retry,
+        record_data=record_data,
+        image_format=image_format,
+        folder_mode=folder_mode,
+    ) as xhs:  # 使用自定义参数
+        download = True  # 是否下载作品文件，默认值：False
+        # 返回作品详细信息，包括下载地址
+        # 获取数据失败时返回空字典
+        print(
+            await xhs.extract(
+                error_link,
+                download,
+            )
+        )
+        print(await xhs.extract(demo_link, download, index=[1, 2]))
+        # 支持传入多个作品链接
+        print(
+            await xhs.extract(
+                multiple_links,
+                download,
+            )
+        )
+
+
+async def app():
+    async with XHSDownloader() as xhs:
+        await xhs.run_async()
+
+
+async def server(
+    host="0.0.0.0",
+    port=8000,
+    log_level="info",
+):
+    async with XHS(**Settings().run()) as xhs:
+        await xhs.run_server(
+            host,
+            port,
+            log_level,
+        )
+
+
+if __name__ == "__main__":
+    with suppress(
+        KeyboardInterrupt,
+        CancelledError,
+    ):
+        if len(argv) == 1:
+            run(app())
+        elif argv[1] == "server":
+            run(server())
+        else:
+            cli()

parsehub/deps/xhs/source/CLI/__init__.py

@@ -0,0 +1,3 @@
+from .main import cli
+
+__all__ = ["cli"]

parsehub/deps/xhs/source/CLI/main.py

@@ -0,0 +1,330 @@
+from asyncio import run
+from contextlib import suppress
+from pathlib import Path as Root
+from textwrap import fill
+
+from click import Context
+from click import (
+    command,
+    option,
+    Path,
+    Choice,
+    pass_context,
+    echo,
+)
+from rich import print
+from rich.panel import Panel
+from rich.table import Table
+
+from source.application import XHS
+from source.expansion import BrowserCookie
+from source.module import (
+    ROOT,
+    PROJECT,
+)
+from source.module import Settings
+from source.module import Translate
+
+__all__ = ["cli"]
+
+
+def check_value(function):
+    def inner(ctx: Context, param, value):
+        if not value:
+            return
+        return function(ctx, param, value)
+
+    return inner
+
+
+class CLI:
+    def __init__(self, ctx: Context, **kwargs):
+        # print(kwargs)
+        self.ctx = ctx
+        self.url = kwargs.pop("url")
+        self.index = self.__format_index(kwargs.pop("index"))
+        self.path = kwargs.pop("settings")
+        self.update = kwargs.pop("update_settings")
+        # print(kwargs)
+        self.settings = Settings(self.__check_settings_path())
+        self.parameter = self.settings.run() | self.__clean_params(kwargs)
+        self.APP = XHS(**self.parameter)
+
+    async def __aenter__(self):
+        await self.APP.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.APP.__aexit__(exc_type, exc_value, traceback)
+
+    async def run(self):
+        if self.url:
+            await self.APP.extract_cli(self.url, index=self.index)
+        self.__update_settings()
+
+    def __update_settings(self):
+        if self.update:
+            self.settings.update(self.parameter)
+
+    def __check_settings_path(self) -> Path:
+        if not self.path:
+            return ROOT
+        return s.parent if (s := Root(self.path)).is_file() else ROOT
+
+    @staticmethod
+    def __merge_cookie(data: dict) -> None:
+        if not data["cookie"] and (bc := data["browser_cookie"]):
+            data["cookie"] = bc
+        data.pop("browser_cookie")
+
+    def __clean_params(self, data: dict) -> dict:
+        self.__merge_cookie(data)
+        return {k: v for k, v in data.items() if v}
+
+    @staticmethod
+    def __format_index(index: str) -> list:
+        if index:
+            result = []
+            values = index.split()
+            for i in values:
+                with suppress(ValueError):
+                    result.append(int(i))
+            return result
+        return []
+
+    @staticmethod
+    @check_value
+    def version(ctx: Context, param, value) -> None:
+        echo(PROJECT)
+        ctx.exit()
+
+    @staticmethod
+    @check_value
+    def read_cookie(ctx: Context, param, value) -> str:
+        return BrowserCookie.get(
+            value,
+            domains=[
+                "xiaohongshu.com",
+            ],
+        )
+
+    @staticmethod
+    @check_value
+    def help_(ctx: Context, param, value) -> None:
+        _ = Translate("").message()
+        table = Table(highlight=True, box=None, show_header=True)
+
+        # 添加表格的列名
+        table.add_column("parameter", no_wrap=True, style="bold")
+        table.add_column("abbreviation", no_wrap=True, style="bold")
+        table.add_column("type", no_wrap=True, style="bold")
+        table.add_column(
+            "description",
+            no_wrap=True,
+        )
+
+        # TODO: 语言设置未生效
+        options = (
+            ("--url", "-u", "str", _("小红书作品链接")),
+            (
+                "--index",
+                "-i",
+                "str",
+                _(
+                    '下载指定序号的图片文件，仅对图文作品生效；多个序号输入示例："1 3 5 7"'
+                ),
+            ),
+            ("--work_path", "-wp", "str", _("作品数据 / 文件保存根路径")),
+            ("--folder_name", "-fn", "str", _("作品文件储存文件夹名称")),
+            ("--name_format", "-nf", "str", _("作品文件名称格式")),
+            # ("--sec_ch_ua", "-su", "str", _("Sec-Ch-Ua")),
+            # ("--sec_ch_ua_platform", "-sp", "str", _("Sec-Ch-Ua-Platform")),
+            ("--user_agent", "-ua", "str", _("User-Agent")),
+            ("--cookie", "-ck", "str", _("小红书网页版 Cookie，无需登录")),
+            ("--proxy", "-p", "str", _("网络代理")),
+            ("--timeout", "-t", "int", _("请求数据超时限制，单位：秒")),
+            (
+                "--chunk",
+                "-c",
+                "int",
+                _("下载文件时，每次从服务器获取的数据块大小，单位：字节"),
+            ),
+            ("--max_retry", "-mr", "int", _("请求数据失败时，重试的最大次数")),
+            ("--record_data", "-rd", "bool", _("是否记录作品数据至文件")),
+            (
+                "--image_format",
+                "-if",
+                "choice",
+                _("图文作品文件下载格式，支持：PNG、WEBP"),
+            ),
+            ("--live_download", "-ld", "bool", _("图文动图文件下载开关")),
+            ("--download_record", "-dr", "bool", _("作品下载记录开关")),
+            (
+                "--folder_mode",
+                "-fm",
+                "bool",
+                _("是否将每个作品的文件储存至单独的文件夹"),
+            ),
+            ("--language", "-l", "choice", _("设置程序语言，目前支持：zh_CN、en_GB")),
+            ("--settings", "-s", "str", _("读取指定配置文件")),
+            (
+                "--browser_cookie",
+                "-bc",
+                "choice",
+                fill(
+                    _(
+                        "从指定的浏览器读取小红书网页版 Cookie，支持：{0}; 输入浏览器名称或序号"
+                    ).format(
+                        ", ".join(
+                            f"{i}: {j}"
+                            for i, j in enumerate(
+                                BrowserCookie.SUPPORT_BROWSER.keys(),
+                                start=1,
+                            )
+                        )
+                    ),
+                    width=40,
+                ),
+            ),
+            ("--update_settings", "-us", "flag", _("是否更新配置文件")),
+            ("--help", "-h", "flag", _("查看详细参数说明")),
+            ("--version", "-v", "flag", _("查看 XHS-Downloader 版本")),
+        )
+
+        for option in options:
+            table.add_row(*option)
+
+        print(
+            Panel(
+                table,
+                border_style="bold",
+                title="XHS-Downloader CLI Parameters",
+                title_align="left",
+            )
+        )
+        ctx.exit()
+
+
+@command(name="XHS-Downloader", help=PROJECT)
+@option(
+    "--url",
+    "-u",
+)
+@option(
+    "--index",
+    "-i",
+)
+@option(
+    "--work_path",
+    "-wp",
+    type=Path(file_okay=False),
+)
+@option(
+    "--folder_name",
+    "-fn",
+)
+@option(
+    "--name_format",
+    "-nf",
+)
+# @option("--sec_ch_ua", "-su", )
+# @option("--sec_ch_ua_platform", "-sp", )
+@option(
+    "--user_agent",
+    "-ua",
+)
+@option(
+    "--cookie",
+    "-ck",
+)
+@option(
+    "--proxy",
+    "-p",
+)
+@option(
+    "--timeout",
+    "-t",
+    type=int,
+)
+@option(
+    "--chunk",
+    "-c",
+    type=int,
+)
+@option(
+    "--max_retry",
+    "-mr",
+    type=int,
+)
+@option(
+    "--record_data",
+    "-rd",
+    type=bool,
+)
+@option(
+    "--image_format",
+    "-if",
+    type=Choice(["png", "PNG", "webp", "WEBP"]),
+)
+@option(
+    "--live_download",
+    "-ld",
+    type=bool,
+)
+@option(
+    "--download_record",
+    "-dr",
+    type=bool,
+)
+@option(
+    "--folder_mode",
+    "-fm",
+    type=bool,
+)
+@option(
+    "--language",
+    "-l",
+    type=Choice(["zh_CN", "en_GB"]),
+)
+@option(
+    "--settings",
+    "-s",
+    type=Path(dir_okay=False),
+)
+@option(
+    "--browser_cookie",
+    "-bc",
+    type=Choice(
+        list(BrowserCookie.SUPPORT_BROWSER.keys())
+        + [str(i) for i in range(1, len(BrowserCookie.SUPPORT_BROWSER) + 1)]
+    ),
+    callback=CLI.read_cookie,
+)
+@option(
+    "--update_settings",
+    "-us",
+    type=bool,
+    is_flag=True,
+)
+@option(
+    "-h",
+    is_flag=True,
+    is_eager=True,
+    expose_value=False,
+    callback=CLI.help_,
+)
+@option(
+    "--version",
+    "-v",
+    is_flag=True,
+    is_eager=True,
+    expose_value=False,
+    callback=CLI.version,
+)
+@pass_context
+def cli(ctx, **kwargs):
+    async def main():
+        async with CLI(ctx, **kwargs) as xhs:
+            await xhs.run()
+
+    run(main())

parsehub/deps/xhs/source/TUI/__init__.py

@@ -0,0 +1,3 @@
+from .app import XHSDownloader
+
+__all__ = ["XHSDownloader"]

parsehub/deps/xhs/source/TUI/about.py

@@ -0,0 +1,76 @@
+from typing import Callable
+
+from rich.text import Text
+from textual.app import ComposeResult
+from textual.binding import Binding
+from textual.screen import Screen
+from textual.widgets import Footer
+from textual.widgets import Header
+from textual.widgets import Label
+
+from ..module import (
+    PROJECT,
+    PROMPT,
+    MASTER,
+    INFO,
+)
+
+__all__ = ["About"]
+
+
+class About(Screen):
+    BINDINGS = [
+        Binding(key="Q", action="quit", description="退出程序/Quit"),
+        Binding(key="U", action="check_update", description="检查更新/Update"),
+        Binding(key="B", action="index", description="返回首页/Back"),
+    ]
+
+    def __init__(self, message: Callable[[str], str]):
+        super().__init__()
+        self.message = message
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield Label(
+            Text(
+                self.message(
+                    "如果 XHS-Downloader 对您有帮助，请考虑为它点个 Star，感谢您的支持！"
+                ),
+                style=INFO,
+            ),
+            classes="prompt",
+        )
+        yield Label(
+            Text("Discord 社区", style=PROMPT),
+            classes="prompt",
+        )
+        yield Label(
+            f"{self.message("邀请链接：")}https://discord.com/invite/ZYtmgKud9Y"
+        )
+        yield Label(
+            Text(self.message("作者的其他开源项目"), style=PROMPT),
+            classes="prompt",
+        )
+        yield Label(
+            Text("TikTokDownloader (抖音 / TikTok)", style=MASTER),
+            classes="prompt",
+        )
+        yield Label("https://github.com/JoeanAmier/TikTokDownloader")
+        yield Label(
+            Text("KS-Downloader (快手)", style=MASTER),
+            classes="prompt",
+        )
+        yield Label("https://github.com/JoeanAmier/KS-Downloader")
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self.title = PROJECT
+
+    async def action_quit(self) -> None:
+        await self.app.action_quit()
+
+    async def action_index(self):
+        await self.app.push_screen("index")
+
+    async def action_check_update(self):
+        await self.app.run_action("update_and_return")