parsehub 1.0.0__tar.gz

parsehub-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 梓澪
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

parsehub-1.0.0/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.1
+Name: parsehub
+Version: 1.0.0
+Summary: 支持AI总结的社交媒体聚合解析器
+Requires-Python: >=3.10.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiocache>=0.12.3
+Requires-Dist: aiofiles>=24.1.0
+Requires-Dist: apscheduler>=4.0.0a5
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: dynamicadaptor>=0.5.2
+Requires-Dist: dynrender-skia>=0.2.5
+Requires-Dist: langchain>=0.2.5
+Requires-Dist: langchain-core>=0.2.8
+Requires-Dist: langchain-openai>=0.1.14
+Requires-Dist: loguru>=0.6.0
+Requires-Dist: openai>=1.54.5
+Requires-Dist: opencv-python>=4.10.0.84
+Requires-Dist: playwright>=1.48.0
+Requires-Dist: pydub>=0.25.1
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: tenacity>=8.5.0
+Requires-Dist: urlextract>=1.9.0
+Requires-Dist: yt-dlp>=2024.11.18.232921.dev0
+Requires-Dist: pydantic>=1.10.19
+Requires-Dist: textual<=0.63.0
+Requires-Dist: pyperclip>=1.9.0
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: click>=8.1.7
+Requires-Dist: rookiepy>=0.5.6
+Requires-Dist: fastapi>=0.112.1
+Requires-Dist: uvicorn>=0.30.6
+Requires-Dist: emoji>=2.14.0
+Requires-Dist: instaloader>=4.14
+
+# ParseHub
+
+**支持AI总结的社交媒体聚合解析器**
+
+> 视频总结会调用 `whisper-1` 模型
+
+```python
+from parsehub import ParseHub
+from parsehub.config import ParseHubConfig
+import asyncio
+
+
+async def main():
+    # ParseHubConfig.api_key = 'your_api_key'
+    # ParseHubConfig.base_url = 'your_base_url'
+
+    ph = ParseHub()
+    result = await ph.parse('https://twitter.com/aobuta_anime/status/1827284717848424696')
+    print(result)
+
+    # download_result = await result.download()
+    # print(download_result.media)
+    # summary_result = await download_result.summary()
+    # await download_result.delete()
+
+    summary_result = await result.summary()
+    print(summary_result.content)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+## 环境变量
+
+| 名称         | 描述                                                                       | 默认值                       |
+|------------|--------------------------------------------------------------------------|---------------------------|
+| DOUYIN_API | 抖音解析API地址, 项目地址: https://github.com/Evil0ctal/Douyin_TikTok_Download_API | https://douyin.wtf        |
+| PROVIDER   | 模型提供商, 暂只支持openai                                                        | openai                    |
+| API_KEY    | API Key                                                                  |                           |
+| BASE_URL   | API 地址                                                                   | https://api.openai.com/v1 |
+| MODEL      | AI总结使用的模型                                                                | gpt-4o-mini               |

parsehub-1.0.0/README.md

@@ -0,0 +1,42 @@
+# ParseHub
+
+**支持AI总结的社交媒体聚合解析器**
+
+> 视频总结会调用 `whisper-1` 模型
+
+```python
+from parsehub import ParseHub
+from parsehub.config import ParseHubConfig
+import asyncio
+
+
+async def main():
+    # ParseHubConfig.api_key = 'your_api_key'
+    # ParseHubConfig.base_url = 'your_base_url'
+
+    ph = ParseHub()
+    result = await ph.parse('https://twitter.com/aobuta_anime/status/1827284717848424696')
+    print(result)
+
+    # download_result = await result.download()
+    # print(download_result.media)
+    # summary_result = await download_result.summary()
+    # await download_result.delete()
+
+    summary_result = await result.summary()
+    print(summary_result.content)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+## 环境变量
+
+| 名称         | 描述                                                                       | 默认值                       |
+|------------|--------------------------------------------------------------------------|---------------------------|
+| DOUYIN_API | 抖音解析API地址, 项目地址: https://github.com/Evil0ctal/Douyin_TikTok_Download_API | https://douyin.wtf        |
+| PROVIDER   | 模型提供商, 暂只支持openai                                                        | openai                    |
+| API_KEY    | API Key                                                                  |                           |
+| BASE_URL   | API 地址                                                                   | https://api.openai.com/v1 |
+| MODEL      | AI总结使用的模型                                                                | gpt-4o-mini               |

parsehub-1.0.0/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "parsehub"
+version = "1.0.0"
+description = "支持AI总结的社交媒体聚合解析器"
+readme = "README.md"
+requires-python = ">=3.10.11"
+dependencies = [
+    "aiocache>=0.12.3",
+    "aiofiles>=24.1.0",
+    "apscheduler>=4.0.0a5",
+    "beautifulsoup4==4.12.3",
+    "dynamicadaptor>=0.5.2",
+    "dynrender-skia>=0.2.5",
+    "langchain>=0.2.5",
+    "langchain-core>=0.2.8",
+    "langchain-openai>=0.1.14",
+    "loguru>=0.6.0",
+    "openai>=1.54.5",
+    "opencv-python>=4.10.0.84",
+    "playwright>=1.48.0",
+    "pydub>=0.25.1",
+    "python-dotenv>=1.0.1",
+    "tenacity>=8.5.0",
+    "urlextract>=1.9.0",
+    "yt-dlp>=2024.11.18.232921.dev0",
+    "pydantic>=1.10.19",
+    "textual<=0.63.0",
+    "pyperclip>=1.9.0",
+    "lxml>=5.3.0",
+    "pyyaml>=6.0.2",
+    "aiosqlite>=0.20.0",
+    "click>=8.1.7",
+    "rookiepy>=0.5.6",
+    "fastapi>=0.112.1",
+    "uvicorn>=0.30.6",
+    "emoji>=2.14.0",
+    "instaloader>=4.14",
+]
+[tool.hatch.build.targets.wheel]
+packages = ["src/parsehub"]

parsehub-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

parsehub-1.0.0/src/parsehub/__init__.py

@@ -0,0 +1,7 @@
+from .deps.submodule_manager import SubmoduleManager
+from .main import ParseHub
+
+manager = SubmoduleManager()
+manager.setup_all()
+
+__all__ = ["ParseHub"]

parsehub-1.0.0/src/parsehub/config/__init__.py

@@ -0,0 +1,3 @@
+from .config import ParseHubConfig
+
+__all__ = ["ParseHubConfig"]

parsehub-1.0.0/src/parsehub/config/config.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+import sys
+from os import getenv
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+
+
+@dataclass
+class ParseHubConfig:
+    DOWNLOAD_DIR = getenv("DOWNLOAD_DIR", Path(sys.argv[0]).parent / Path("downloads/"))
+    """默认下载目录"""
+
+    douyin_api = getenv("DOUYIN_API", "https://douyin.wtf")
+    """抖音解析API地址, 项目地址: https://github.com/Evil0ctal/Douyin_TikTok_Download_API"""
+
+    provider = getenv("PROVIDER", "openai").lower()
+    """模型提供商"""
+
+    api_key = getenv("API_KEY")
+    """API Key"""
+
+    base_url = getenv("BASE_URL", "https://api.openai.com/v1")
+    """API 地址"""
+
+    model = getenv("MODEL", "gpt-4o-mini")
+    """AI总结模型名称"""

parsehub-1.0.0/src/parsehub/deps/submodule_manager.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+import sys
+from typing import Dict, Optional
+
+
+class SubmoduleManager:
+    def __init__(self):
+        self.deps_dir = Path(__file__).parent
+        self.path_mappings: Dict[str, Path] = {}
+
+    def add_submodule(
+        self, name: str, relative_path: str, source_dir: Optional[str] = None
+    ):
+        """添加子模块路径映射"""
+        submodule_path = self.deps_dir / relative_path
+        self.path_mappings[name] = submodule_path
+
+        # 添加主模块路径
+        if str(submodule_path) not in sys.path:
+            sys.path.insert(0, str(submodule_path))
+
+        # 添加源码目录路径（如果指定）
+        if source_dir:
+            source_path = submodule_path / source_dir
+            if str(source_path) not in sys.path:
+                sys.path.insert(0, str(source_path))
+
+    def setup_all(self):
+        """设置所有子模块的路径"""
+        # 添加所有需要的子模块
+        self.add_submodule("xhs", "xhs", "xhs")

parsehub-1.0.0/src/parsehub/deps/xhs/locale/po_to_mo.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from subprocess import run
+
+__all__ = []
+ROOT = Path(__file__).resolve().parent
+
+
+def scan_directory():
+    return [
+        item.joinpath("LC_MESSAGES/xhs.po") for item in ROOT.iterdir() if item.is_dir()
+    ]
+
+
+def generate_map(files: list[Path]):
+    return [(i, i.with_suffix(".mo")) for i in files]
+
+
+def generate_mo(maps: list[tuple[Path, Path]]):
+    for i, j in maps:
+        command = f'msgfmt "{i}" -o "{j}"'
+        print(run(command, shell=True, text=True))
+
+
+if __name__ == "__main__":
+    generate_mo(generate_map(scan_directory()))

parsehub-1.0.0/src/parsehub/deps/xhs/main.py

@@ -0,0 +1,94 @@
+from asyncio import run
+from asyncio.exceptions import CancelledError
+from contextlib import suppress
+from sys import argv
+
+from source import Settings
+from source import XHS
+from source import XHSDownloader
+from source import cli
+
+
+async def example():
+    """通过代码设置参数，适合二次开发"""
+    # 示例链接
+    error_link = "https://github.com/JoeanAmier/XHS_Downloader"
+    demo_link = "https://www.xiaohongshu.com/explore/xxxxxxxxxx"
+    multiple_links = f"{demo_link} {demo_link} {demo_link}"
+    # 实例对象
+    work_path = "D:\\"  # 作品数据/文件保存根路径，默认值：项目根路径
+    folder_name = "Download"  # 作品文件储存文件夹名称（自动创建），默认值：Download
+    name_format = "作品标题 作品描述"
+    user_agent = ""  # User-Agent
+    cookie = ""  # 小红书网页版 Cookie，无需登录，可选参数，登录状态对数据采集有影响
+    proxy = None  # 网络代理
+    timeout = 5  # 请求数据超时限制，单位：秒，默认值：10
+    chunk = 1024 * 1024 * 10  # 下载文件时，每次从服务器获取的数据块大小，单位：字节
+    max_retry = 2  # 请求数据失败时，重试的最大次数，单位：秒，默认值：5
+    record_data = False  # 是否保存作品数据至文件
+    image_format = "WEBP"  # 图文作品文件下载格式，支持：PNG、WEBP
+    folder_mode = False  # 是否将每个作品的文件储存至单独的文件夹
+    # async with XHS() as xhs:
+    #     pass  # 使用默认参数
+    async with XHS(
+        work_path=work_path,
+        folder_name=folder_name,
+        name_format=name_format,
+        user_agent=user_agent,
+        cookie=cookie,
+        proxy=proxy,
+        timeout=timeout,
+        chunk=chunk,
+        max_retry=max_retry,
+        record_data=record_data,
+        image_format=image_format,
+        folder_mode=folder_mode,
+    ) as xhs:  # 使用自定义参数
+        download = True  # 是否下载作品文件，默认值：False
+        # 返回作品详细信息，包括下载地址
+        # 获取数据失败时返回空字典
+        print(
+            await xhs.extract(
+                error_link,
+                download,
+            )
+        )
+        print(await xhs.extract(demo_link, download, index=[1, 2]))
+        # 支持传入多个作品链接
+        print(
+            await xhs.extract(
+                multiple_links,
+                download,
+            )
+        )
+
+
+async def app():
+    async with XHSDownloader() as xhs:
+        await xhs.run_async()
+
+
+async def server(
+    host="0.0.0.0",
+    port=8000,
+    log_level="info",
+):
+    async with XHS(**Settings().run()) as xhs:
+        await xhs.run_server(
+            host,
+            port,
+            log_level,
+        )
+
+
+if __name__ == "__main__":
+    with suppress(
+        KeyboardInterrupt,
+        CancelledError,
+    ):
+        if len(argv) == 1:
+            run(app())
+        elif argv[1] == "server":
+            run(server())
+        else:
+            cli()

parsehub-1.0.0/src/parsehub/deps/xhs/source/CLI/__init__.py

@@ -0,0 +1,3 @@
+from .main import cli
+
+__all__ = ["cli"]