wxmp 1.0.0__tar.gz

wxmp-1.0.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.3
+Name: wxmp
+Version: 1.0.0
+Summary: 微信公众号 API 等相关工具
+Author: morning-start
+Author-email: morning-start <morning-start@foxmail.com>
+Requires-Dist: fake-useragent>=2.2.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: requests>=2.32.5
+Requires-Dist: tqdm>=4.67.3
+Requires-Dist: urllib3>=2.6.3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# 微信公众号 API 工具
+
+微信公众平台（微信公众号）API 相关工具，提供公众号搜索、文章列表获取、文章内容下载等功能。
+
+**项目地址**: https://github.com/morning-start/wxmp
+
+## 项目简介
+
+`wxmp` 是一个 Python 库，用于与微信公众平台后台 API 进行交互。通过本项目可以：
+
+- 通过微信登录后的 cookies 获取访问 token
+- 搜索公众号（根据关键词查找公众号信息）
+- 获取指定公众号的文章列表
+- 验证文章链接有效性
+- 下载文章内容并转换为 Markdown 格式
+- 支持时间范围缓存和增量更新
+- 支持并发下载
+
+## 安装
+
+```bash
+pip install wxmp
+```
+
+或从源码安装：
+
+```bash
+git clone https://github.com/morning-start/wxmp.git
+cd wxmp
+pip install -e .
+```
+
+## 快速开始
+
+### 1. 初始化 API
+
+```python
+from wxmp import WxMPAPI
+
+cookies = {
+    "wxuin": "your_wxuin",
+    "pass_ticket": "your_pass_ticket",
+}
+
+api = WxMPAPI(cookies)
+```
+
+### 2. 搜索公众号
+
+```python
+response = api.search_fakeid("Python")
+for account in response.list:
+    print(f"名称: {account.nickname}")
+```
+
+### 3. 使用时间范围爬虫（推荐）
+
+```python
+from wxmp.spider import TimeRangeSpider
+from datetime import datetime
+
+spider = TimeRangeSpider.from_cookies_file("cookies.json")
+bizs = spider.load_or_search_bizs(["Python编程"])
+
+time_range = TimeRange(
+    begin=datetime(2024, 1, 1),
+    end=datetime(2024, 12, 31)
+)
+
+df = spider.search_articles_content(bizs, time_range)
+spider.save_all_article_content(df, save_dir="temp/article_content/")
+```
+
+## 文档
+
+详细文档请查看 [Wiki](./wiki/README.md)：
+
+- [项目概览](./wiki/项目概览.md) - 项目简介、技术栈、架构概览
+- [架构设计](./wiki/架构设计.md) - 设计原则、模块设计、缓存策略
+- [API 文档](./wiki/API文档.md) - API 层完整文档、数据模型、异常类
+- [使用指南](./wiki/使用指南.md) - 快速开始、使用场景、最佳实践
+- [数据流动与状态管理](./wiki/数据流动与状态管理.md) - 数据流转、状态机、缓存策略
+- [贡献指南](./wiki/贡献指南.md) - 如何贡献代码、开发环境设置
+- [常见问题](./wiki/常见问题.md) - 常见问题和解决方案
+- [更新日志](./wiki/CHANGELOG.md) - 版本历史、变更记录
+
+## License
+
+MIT License

wxmp-1.0.0/README.md

@@ -0,0 +1,89 @@
+# 微信公众号 API 工具
+
+微信公众平台（微信公众号）API 相关工具，提供公众号搜索、文章列表获取、文章内容下载等功能。
+
+**项目地址**: https://github.com/morning-start/wxmp
+
+## 项目简介
+
+`wxmp` 是一个 Python 库，用于与微信公众平台后台 API 进行交互。通过本项目可以：
+
+- 通过微信登录后的 cookies 获取访问 token
+- 搜索公众号（根据关键词查找公众号信息）
+- 获取指定公众号的文章列表
+- 验证文章链接有效性
+- 下载文章内容并转换为 Markdown 格式
+- 支持时间范围缓存和增量更新
+- 支持并发下载
+
+## 安装
+
+```bash
+pip install wxmp
+```
+
+或从源码安装：
+
+```bash
+git clone https://github.com/morning-start/wxmp.git
+cd wxmp
+pip install -e .
+```
+
+## 快速开始
+
+### 1. 初始化 API
+
+```python
+from wxmp import WxMPAPI
+
+cookies = {
+    "wxuin": "your_wxuin",
+    "pass_ticket": "your_pass_ticket",
+}
+
+api = WxMPAPI(cookies)
+```
+
+### 2. 搜索公众号
+
+```python
+response = api.search_fakeid("Python")
+for account in response.list:
+    print(f"名称: {account.nickname}")
+```
+
+### 3. 使用时间范围爬虫（推荐）
+
+```python
+from wxmp.spider import TimeRangeSpider
+from datetime import datetime
+
+spider = TimeRangeSpider.from_cookies_file("cookies.json")
+bizs = spider.load_or_search_bizs(["Python编程"])
+
+time_range = TimeRange(
+    begin=datetime(2024, 1, 1),
+    end=datetime(2024, 12, 31)
+)
+
+df = spider.search_articles_content(bizs, time_range)
+spider.save_all_article_content(df, save_dir="temp/article_content/")
+```
+
+## 文档
+
+详细文档请查看 [Wiki](./wiki/README.md)：
+
+- [项目概览](./wiki/项目概览.md) - 项目简介、技术栈、架构概览
+- [架构设计](./wiki/架构设计.md) - 设计原则、模块设计、缓存策略
+- [API 文档](./wiki/API文档.md) - API 层完整文档、数据模型、异常类
+- [使用指南](./wiki/使用指南.md) - 快速开始、使用场景、最佳实践
+- [数据流动与状态管理](./wiki/数据流动与状态管理.md) - 数据流转、状态机、缓存策略
+- [贡献指南](./wiki/贡献指南.md) - 如何贡献代码、开发环境设置
+- [常见问题](./wiki/常见问题.md) - 常见问题和解决方案
+- [更新日志](./wiki/CHANGELOG.md) - 版本历史、变更记录
+
+## License
+
+MIT License

wxmp-1.0.0/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "wxmp"
+version = "1.0.0"
+description = "微信公众号 API 等相关工具"
+readme = "README.md"
+authors = [
+    { name = "morning-start", email = "morning-start@foxmail.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "fake-useragent>=2.2.0",
+    "loguru>=0.7.3",
+    "pandas>=2.3.3",
+    "pydantic>=2.12.5",
+    "requests>=2.32.5",
+    "tqdm>=4.67.3",
+    "urllib3>=2.6.3",
+]
+
+[build-system]
+requires = ["uv_build>=0.10.4,<0.11.0"]
+build-backend = "uv_build"
+
+[[tool.uv.index]]
+url = "https://pypi.tuna.tsinghua.edu.cn/simple"
+default = true

wxmp-1.0.0/src/wxmp/__init__.py

@@ -0,0 +1,22 @@
+from .api import WxMPAPI
+from .api.common import WxMPAPIError
+from .api.list_ex import ArticleListItem, ListExError, ListExRequest, ListExResponse
+from .api.search_biz import SearchBizError, SearchBizRequest, SearchBizResponse
+from .api.token import TokenError, TokenResponse
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "WxMPAPI",
+    "WxMPAPIError",
+    "TokenError",
+    "TokenResponse",
+    "SearchBizError",
+    "SearchBizRequest",
+    "SearchBizResponse",
+    "ListExError",
+    "ListExRequest",
+    "ListExResponse",
+    "ArticleListItem",
+]

wxmp-1.0.0/src/wxmp/api/__init__.py

@@ -0,0 +1,18 @@
+from .index import WxMPAPI
+from .list_ex import ArticleListItem, ListExError, ListExRequest, ListExResponse
+from .search_biz import SearchBizError, SearchBizRequest, SearchBizResponse
+from .token import TokenError, TokenResponse
+
+__all__ = [
+    "TokenResponse",
+    "SearchBizResponse",
+    "ListExResponse",
+    "SearchBizRequest",
+    "ListExRequest",
+    "WxMPAPI",
+    "WxMPAPIError",
+    "TokenError",
+    "SearchBizError",
+    "ListExError",
+    "ArticleListItem",
+]

wxmp-1.0.0/src/wxmp/api/common.py

@@ -0,0 +1,47 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class WxMPAPIError(Exception):
+    """微信MP API基础异常类"""
+
+    pass
+
+
+class BaseRequest(BaseModel):
+    """API基础请求参数"""
+
+    token: str = Field(description="授权token")
+    begin: int = Field(default=0, description="列表起始位置")
+    count: int = Field(default=5, description="返回消息条数，一个消息可能包含多个文章")
+    lang: str = Field(default="zh_CN", description="语言")
+    f: str = Field(default="json", description="数据格式")
+    ajax: int = Field(default=1, description="是否为ajax请求")
+
+
+class BaseResp(BaseModel):
+    """基础响应对象（微信API标准格式）"""
+
+    ret: int = Field(description="返回码，0表示成功")
+    err_msg: str = Field(default="ok", description="返回信息")
+
+
+class BaseResponse(BaseModel):
+    """API基础响应模型（带base_resp嵌套格式）"""
+
+    base_resp: BaseResp = Field(description="基础响应对象")
+
+
+class ErrorDetail(BaseModel):
+    """错误详情模型"""
+
+    code: int = Field(description="错误码")
+    message: str = Field(description="错误信息")
+    details: Optional[str] = Field(default=None, description="错误详情")
+
+
+class ErrorResponse(BaseModel):
+    """错误响应模型"""
+
+    base_resp: ErrorDetail = Field(description="基础响应")

wxmp-1.0.0/src/wxmp/api/index.py

@@ -0,0 +1,135 @@
+import asyncio
+import re
+import warnings
+
+import requests
+from fake_useragent import UserAgent
+from tqdm.asyncio import tqdm as tqdm_asyncio
+from urllib3.exceptions import InsecureRequestWarning
+
+from .list_ex import ListExError, ListExRequest, ListExResponse
+from .search_biz import SearchBizError, SearchBizRequest, SearchBizResponse
+from .token import TokenError
+
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+class WxMPAPI:
+    def __init__(self, cookies: dict) -> None:
+        self.cookies = cookies
+        self.domain = "https://mp.weixin.qq.com"
+        self.headers = {
+            "User-Agent": UserAgent().random,
+            "Host": "mp.weixin.qq.com",
+            "Referer": "https://mp.weixin.qq.com/",
+        }
+        self.session = requests.Session()
+        self.token = None
+
+    def _fetch_token(self) -> str:
+        url = self.domain
+        try:
+            res = self.session.get(
+                url=url, headers=self.headers, cookies=self.cookies, verify=False
+            )
+            res.raise_for_status()
+
+            token = re.findall(r".*?token=(\d+)", res.url)
+            if token:
+                return token[0]
+            raise TokenError("从重定向URL中提取token失败")
+        except requests.HTTPError as e:
+            raise TokenError(f"HTTP请求失败: {e.response.status_code}")
+        except Exception as e:
+            raise TokenError(f"获取token时发生错误: {str(e)}")
+
+    def fetch_fakeid(
+        self, query: str, begin: int = 0, count: int = 5
+    ) -> SearchBizResponse:
+        url = self.domain + "/cgi-bin/searchbiz"
+        params = SearchBizRequest(
+            action="search_biz",
+            begin=begin,
+            count=count,
+            query=query,
+            token=self.token,
+        )
+        try:
+            res = self.session.get(
+                url=url,
+                params=params.model_dump(),
+                headers=self.headers,
+                cookies=self.cookies,
+                verify=False,
+            )
+            res.raise_for_status()
+            return SearchBizResponse(**res.json())
+        except requests.HTTPError as e:
+            raise SearchBizError(f"HTTP请求失败: {e.response.status_code}")
+        except Exception as e:
+            raise SearchBizError(f"搜索公众号时发生错误: {str(e)}")
+
+    def fetch_article_list(
+        self, fakeid: str, begin: int = 0, count: int = 5
+    ) -> ListExResponse:
+        url = self.domain + "/cgi-bin/appmsg"
+        params = ListExRequest(
+            begin=begin,
+            count=count,
+            fakeid=fakeid,
+            token=self.token,
+        )
+        try:
+            res = self.session.get(
+                url=url,
+                params=params.model_dump(),
+                headers=self.headers,
+                cookies=self.cookies,
+                verify=False,
+            )
+            res.raise_for_status()
+            return ListExResponse(**res.json())
+        except requests.HTTPError as e:
+            raise ListExError(f"HTTP请求失败: {e.response.status_code}")
+        except Exception as e:
+            raise ListExError(f"获取文章列表时发生错误: {str(e)}")
+
+    @staticmethod
+    def is_valid_article_link(link: str) -> bool:
+        """
+        判断文章链接是否有效
+        包含 tempkey= 的链接说明文章已删除或失效
+        """
+        if not link:
+            return False
+        # 检查是否包含 tempkey= 参数（说明文章已失效）
+        if "tempkey=" in link:
+            return False
+        return True
+
+    @staticmethod
+    def fetch_article_content(link: str, timeout: int = 10) -> str:
+        headers = {"User-Agent": UserAgent().random}
+        response = requests.get(link, headers=headers, timeout=timeout)
+        response.raise_for_status()
+        return response.text
+
+    @staticmethod
+    async def fetch_multi_article_content(
+        links: list[str], timeout: int = 10
+    ) -> list[str]:
+        """
+        搜索多篇文章内容, 并返回同顺序内容列表
+        """
+
+        async def fetch_single(link: str) -> str:
+            headers = {"User-Agent": UserAgent().random}
+            response = requests.get(link, headers=headers, timeout=timeout)
+            response.raise_for_status()
+            return response.text
+
+        tasks = [asyncio.to_thread(fetch_single, link) for link in links]
+
+        results = await tqdm_asyncio.gather(*tasks, desc="获取文章内容", unit="篇")
+
+        return results

wxmp-1.0.0/src/wxmp/api/list_ex.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from typing import List
+
+from pydantic import BaseModel, Field, field_serializer
+
+from .common import BaseRequest, BaseResponse, WxMPAPIError
+
+
+class ListExError(WxMPAPIError):
+    """获取文章列表失败异常"""
+
+    pass
+
+
+class ListExRequest(BaseRequest):
+    """获取文章列表请求参数"""
+
+    query: str = Field(default="", description="查询字符串")
+    action: str = Field(default="list_ex", description="动作")
+    fakeid: str = Field(description="公众号ID")
+    type: int = Field(default=9, description="类型")
+
+
+class ArticleListItem(BaseModel):
+    """文章列表项"""
+
+    aid: str = Field(description="文章ID（格式：{appmsgid}_{idx}）")
+    appmsgid: int = Field(description="文章消息ID")
+    cover: str = Field(description="封面图URL")
+    create_time: int = Field(description="创建时间戳")
+    digest: str = Field(description="文章摘要")
+    is_pay_subscribe: int = Field(default=0, description="是否付费订阅")
+    item_show_type: int = Field(default=0, description="展示类型")
+    itemidx: int = Field(description="文章索引")
+    link: str = Field(description="文章链接")
+    tagid: List[str] = Field(default_factory=list, description="标签ID列表")
+    title: str = Field(description="文章标题")
+    update_time: int = Field(description="更新时间戳")
+
+    @field_serializer("create_time", "update_time")
+    def serialize_timestamp(self, value: int) -> str:
+        """将时间戳转换为格式化的日期时间字符串"""
+        return datetime.fromtimestamp(value).strftime("%Y-%m-%d %H:%M:%S")
+
+    @field_serializer("tagid")
+    def serialize_tagid(self, value: List[str]) -> str:
+        """将标签ID列表转换为逗号分隔的字符串"""
+        return ",".join(value) if value else ""
+
+
+class ListExResponse(BaseResponse):
+    """文章列表API响应（list_ex）"""
+
+    app_msg_cnt: int = Field(description="文章总数")
+    app_msg_list: List[ArticleListItem] = Field(description="文章列表")

wxmp-1.0.0/src/wxmp/api/search_biz.py

@@ -0,0 +1,37 @@
+from typing import List
+
+from pydantic import BaseModel, Field
+
+from .common import BaseRequest, BaseResponse, WxMPAPIError
+
+
+class SearchBizError(WxMPAPIError):
+    """搜索公众号失败异常"""
+
+    pass
+
+
+class SearchBizRequest(BaseRequest):
+    """搜索公众号请求参数"""
+
+    query: str = Field(description="查询字符串")
+    action: str = Field(default="search_biz", description="动作")
+
+
+class AccountInfo(BaseModel):
+    """公众号信息"""
+
+    fakeid: str = Field(description="公众号fakeid（用于获取文章列表）")
+    nickname: str = Field(description="公众号名称")
+    alias: str = Field(default="", description="公众号微信号")
+    round_head_img: str = Field(description="公众号头像URL")
+    service_type: int = Field(description="服务类型")
+    signature: str = Field(description="公众号签名")
+    verify_status: int = Field(description="验证状态")
+
+
+class SearchBizResponse(BaseResponse):
+    """搜索公众号API响应"""
+
+    arr: List[AccountInfo] = Field(description="公众号列表")
+    total: int = Field(description="搜索结果总数")

wxmp-1.0.0/src/wxmp/api/token.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+
+from .common import BaseResponse, WxMPAPIError
+
+
+class TokenError(WxMPAPIError):
+    """Token获取失败异常"""
+
+    pass
+
+
+class TokenResponse(BaseResponse):
+    """Token获取响应"""
+
+    redirect_url: str = Field(description="重定向URL（包含token）")

wxmp-1.0.0/src/wxmp/spider/__init__.py

@@ -0,0 +1,3 @@
+from .time_range_spider import TimeRangeSpider
+
+__all__ = ["TimeRangeSpider"]

wxmp-1.0.0/src/wxmp/spider/time_range_spider.py

@@ -0,0 +1,433 @@
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from pathlib import Path
+from typing import Literal, NamedTuple
+
+import pandas as pd
+from loguru import logger
+from pydantic import BaseModel, field_serializer
+from tqdm import tqdm
+
+from wxmp.api import ArticleListItem, SearchBizError, TokenError, WxMPAPI
+from wxmp.tools import load_json, sanitize_filename, save_article_content, save_json
+
+
+class ArticleDownloadTask(NamedTuple):
+    url: str
+    title: str
+    save_dir: Path
+    save_file: Literal["md", "html"] = "md"
+    max_retries: int = 3
+    timeout: int = 30
+    date_str: str = ""
+    account_name: str = ""
+    digest: str = ""
+    min_file_size_kb: int = 3
+
+
+class TimeRange(BaseModel):
+    """文章时间范围，用于缓存管理"""
+
+    begin: datetime
+    end: datetime = datetime.today()
+
+    @field_serializer("begin", "end")
+    def serialize_datetime(self, dt: datetime) -> str:
+        """将 datetime 对象序列化为 YYYY-MM-DD 格式字符串"""
+        return dt.strftime("%Y-%m-%d")
+
+
+class TimeRangeSpider(WxMPAPI):
+    def __init__(self, cookies: dict[str, str]) -> None:
+        super().__init__(cookies)
+        try:
+            self.token = self._fetch_token()
+            logger.info(f"获取token成功: {self.token}")
+        except TokenError as e:
+            logger.error(f"❌ 获取token失败: {str(e)}")
+            raise
+
+    @classmethod
+    def from_cookies_file(cls, file_path: str) -> "TimeRangeSpider":
+        data = load_json(file_path)
+        cookies = data["请求 Cookie"]
+        return cls(cookies)
+
+    def load_or_search_bizs(
+        self, gzh_names: list[str] = None, cache_file: Path = Path("temp/fakeids.json")
+    ) -> dict[str, str]:
+        """
+        加载或获取公众号fakeid映射（带缓存优化）
+
+        Args:
+            gzh_names: 公众号名称列表，如果为空则使用缓存文件中的所有公众号
+            cache_file: 缓存文件路径
+
+        Returns:
+            公众号名称到fakeid的映射
+        """
+        bizs = {}
+
+        if cache_file.exists():
+            logger.info(f"从缓存加载fakeids: {cache_file}")
+            cached_bizs = load_json(cache_file)
+
+            # 如果公众号名称列表为空，则使用缓存文件中的所有公众号
+            if not gzh_names:
+                bizs = cached_bizs
+                logger.info(f"使用缓存文件中的所有公众号，共 {len(bizs)} 个")
+            else:
+                required_names = set(gzh_names)
+                required_cached_bizs = {
+                    name: cached_bizs[name]
+                    for name in required_names & set(cached_bizs.keys())
+                }
+                bizs.update(required_cached_bizs)
+        else:
+            if not gzh_names:
+                logger.warning("缓存文件不存在且未指定公众号名称，返回空字典")
+                return {}
+
+        need_names = set(gzh_names) - set(bizs.keys()) if gzh_names else set()
+        if need_names:
+            logger.info(f"从网络获取fakeids: {need_names}")
+            for name in need_names:
+                try:
+                    result = self.fetch_fakeid(name)
+                    if result.arr:
+                        nickname = result.arr[0].nickname
+                        fakeid = result.arr[0].fakeid
+                        bizs[nickname] = fakeid
+                        logger.info(f"成功获取公众号: {nickname} -> {fakeid}")
+                    else:
+                        logger.warning(f"公众号搜索结果为空: {name}")
+                except SearchBizError as e:
+                    logger.error(f"搜索公众号失败: {name}, 错误: {str(e)}")
+            save_json(bizs, cache_file)
+
+        return bizs
+
+    def search_article_list(
+        self, fakeid: str, begin: int, count: int
+    ) -> list[ArticleListItem]:
+        """
+        获取文章列表
+
+        Args:
+            fakeid: 公众号fakeid
+            begin: 列表起始位置
+            count: 返回数量
+
+        Returns:
+            过滤后的文章列表
+        """
+        articles = self.fetch_article_list(fakeid, begin, count)
+        valid_articles = [
+            article
+            for article in articles.app_msg_list
+            if self.is_valid_article_link(article.link)
+        ]
+        return valid_articles
+
+    def search_articles(
+        self,
+        fakeid: str,
+        max_count: int | None = None,
+        time_range: TimeRange | None = None,
+    ) -> list[ArticleListItem]:
+        """
+        加载或获取文章链接列表（带缓存优化）
+
+        Args:
+            nickname: 公众号名称
+            fakeid: 公众号fakeid
+            max_count: 最大获取数量限制
+            time_range: 时间范围限制
+
+        Returns:
+            文章链接列表
+        """
+        EACH_COUNT = 5
+        # article 中有时间属性，获取所有在时间范围的文章信息，
+        all_articles: list[ArticleListItem] = []
+        begin = 0
+        while True:
+            # 获取到的文章都是倒序排列，就是由近到远的顺序，越在后面的越早
+            articles = self.search_article_list(fakeid, begin, EACH_COUNT)
+            all_articles += articles
+            begin += EACH_COUNT
+            # 如果articles为空list，说明超出范围，停止获取
+            if not articles:
+                logger.warning(f"公众号「{nickname}」获取到的文章为空，停止获取")
+                break
+            # 如果时间范围限制存在，超过start_date，停止获取
+            if time_range and articles[-1].create_time < time_range.begin.timestamp():
+                break
+            # 如果最大数量限制存在，超过最大数量，停止获取
+            if max_count and len(all_articles) >= max_count:
+                logger.warning(
+                    f"公众号「{nickname}」获取到的文章数量 {len(all_articles)} 已超过最大数量 {max_count}，停止获取"
+                )
+                break
+
+        return all_articles
+
+    # 剩余的时间范围 meta_file,start_time,end_time
+    @staticmethod
+    def get_remaining_time_range(
+        meta_file: Path, need_time: TimeRange
+    ) -> tuple[TimeRange, TimeRange]:
+        """
+        获取剩余的时间范围
+
+        Args:
+            meta_file: 元数据文件路径
+            time_range: 时间范围
+
+        Returns:
+            剩余的时间范围（开始日期，结束日期）
+        """
+        if not meta_file.exists():
+            return need_time, need_time
+        meta_time = TimeRange(**load_json(meta_file))
+
+        remaining_range, new_meta_info = match_remaining_time_range(
+            meta_time, need_time
+        )
+        return remaining_range, new_meta_info
+
+    def search_articles_content(
+        self,
+        bizs: dict[str, str],
+        time_range: TimeRange,
+        save_dir: Path = Path("temp/articles_info/"),
+    ) -> pd.DataFrame:
+        """
+        获取文章内容（不带缓存优化）
+
+        Args:
+            bizs: 公众号名称到fakeid的映射
+            time_range: 时间范围
+
+        Returns:
+            文章内容DataFrame
+        """
+        # 创建保存目录
+        save_dir.mkdir(parents=True, exist_ok=True)
+
+        for nickname, fakeid in bizs.items():
+            safe_nickname = sanitize_filename(nickname)
+            save_path = save_dir / f"{safe_nickname}.csv"
+            meta_path = save_dir / f"{safe_nickname}.json"
+            remaining_range, new_meta_info = self.get_remaining_time_range(
+                meta_path, time_range
+            )
+            if remaining_range is None:
+                # debug剩余时间，检查是否计算正确
+                logger.debug(
+                    f"公众号 {nickname} 剩余时间范围 {remaining_range}，元数据 {new_meta_info}"
+                )
+                logger.info(f"公众号 {nickname} 已经获取到所有文章，跳过")
+                continue
+            articles = self.search_articles(fakeid, time_range=remaining_range)
+            if not articles:
+                logger.warning(f"公众号 {nickname} 没有获取到有效文章")
+                continue
+
+            df_articles = pd.DataFrame([article.model_dump() for article in articles])
+
+            # 如果文件存在，则合并
+            if save_path.exists():
+                df_existing = pd.read_csv(save_path)
+                df_articles = pd.concat([df_articles, df_existing], ignore_index=True)
+
+            # 去重
+            df_articles = df_articles.drop_duplicates(
+                subset=["title"], keep="first", ignore_index=True
+            )
+            # 按照时间排序
+            df_articles["create_time"] = pd.to_datetime(df_articles["create_time"])
+            df_articles = df_articles.sort_values(
+                by="create_time", ascending=False, ignore_index=True
+            )
+
+            # 保存到缓存文件
+            df_articles.to_csv(save_path, index=False, encoding="utf-8-sig")
+            # 保存元数据
+            save_json(new_meta_info.model_dump(), meta_path)
+        # 合并bizs中对应的csv文件，并且 nickname 列为对应公众号名称
+        csv_files: list[Path] = []
+        for nickname in bizs.keys():
+            safe_nickname = sanitize_filename(nickname)
+            csv_path = save_dir / f"{safe_nickname}.csv"
+            if csv_path.exists():
+                csv_files.append(csv_path)
+        if csv_files:
+            df = pd.concat(
+                [pd.read_csv(f).assign(nickname=f.stem) for f in csv_files],
+                ignore_index=True,
+            )
+        else:
+            df = pd.DataFrame()
+        return df
+
+    @staticmethod
+    def download_article_content(task: ArticleDownloadTask) -> bool:
+        """
+        保存文章内容到Markdown文件
+
+        Args:
+            task: ArticleDownloadTask 文章下载任务
+
+        Returns:
+            是否成功保存
+        """
+        max_retries = task.max_retries
+
+        task.save_dir.mkdir(parents=True, exist_ok=True)
+
+        safe_title = sanitize_filename(task.title)
+        save_path = task.save_dir / f"{safe_title}.{task.save_file}"
+
+        if save_path.exists():
+            return True
+
+        for attempt in range(max_retries):
+            try:
+                content = WxMPAPI.fetch_article_content(task.url, timeout=task.timeout)
+                result = save_article_content(
+                    content,
+                    save_path,
+                    task.save_file,
+                    title=task.title,
+                    date_str=task.date_str,
+                    link=task.url,
+                    account_name=task.account_name,
+                    digest=task.digest,
+                    min_file_size_kb=task.min_file_size_kb,
+                )
+                return result
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    logger.error(
+                        f"获取文章内容失败（重试{max_retries}次后）: {task.title}, 错误: {e}"
+                    )
+                    return False
+                time.sleep(1)
+        return False
+
+    @staticmethod
+    def save_all_article_content(
+        df: pd.DataFrame,
+        save_dir: Path = Path("temp/article_content/"),
+        max_workers: int = 5,
+        time_range: TimeRange = None,
+        save_file: Literal["md", "html"] = "md",
+        min_file_size_kb: int = 3,
+    ):
+        """
+        保存所有文章内容到Markdown文件（并发下载）
+
+        Args:
+            df: 包含文章信息的DataFrame
+            save_dir: 保存目录
+            max_workers: 最大并发数
+            time_range: 时间范围
+            save_file: 保存格式（md 或 html）
+            min_file_size_kb: 最小文件大小（KB）
+        """
+        save_dir.mkdir(parents=True, exist_ok=True)
+        # 筛选出在时间范围内的文章
+        df["create_time"] = pd.to_datetime(df["create_time"])
+        if time_range:
+            df = df[
+                (df["create_time"] >= time_range.begin)
+                & (df["create_time"] <= time_range.end)
+            ]
+
+        tasks = []
+        for _, row in df.iterrows():
+            safe_nickname = sanitize_filename(row["nickname"])
+            task = ArticleDownloadTask(
+                url=row["link"],
+                title=row["title"],
+                save_dir=save_dir / safe_nickname,
+                save_file=save_file,
+                max_retries=3,
+                timeout=30,
+                date_str=row.get("create_time", ""),
+                account_name=row.get("nickname", ""),
+                digest=row.get("digest", ""),
+                min_file_size_kb=min_file_size_kb,
+            )
+            tasks.append((task, row["link"], row["title"]))
+
+        success_count = 0
+        fail_count = 0
+        skip_count = 0
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(TimeRangeSpider.download_article_content, task): (
+                    url,
+                    title,
+                )
+                for task, url, title in tasks
+            }
+
+            with tqdm(total=len(futures), desc="下载文章", unit="篇") as pbar:
+                for future in as_completed(futures):
+                    url, title = futures[future]
+                    try:
+                        result = future.result()
+                        if result:
+                            success_count += 1
+                        else:
+                            fail_count += 1
+                    except Exception as e:
+                        fail_count += 1
+                        logger.error(f"处理文章时发生异常: {title}, 错误: {e}")
+                    pbar.update(1)
+
+        logger.info(
+            f"文章下载完成: 成功 {success_count} 篇, 失败 {fail_count} 篇, "
+            f"跳过 {skip_count} 篇, 总计 {len(tasks)} 篇"
+        )
+
+
+def match_remaining_time_range(
+    meta_time: TimeRange, need_time: TimeRange
+) -> tuple[TimeRange, TimeRange]:
+    """
+    获取剩余的时间范围
+
+    Args:
+        meta_time: 元数据时间范围
+        need_time: 需要获取的时间范围
+
+    Returns:
+        remaining_range: 剩余的时间范围
+        meta_time: 更新后的元数据时间范围
+    """
+
+    # 情况1: 完全没有重叠
+    # 情况2: 缓存在请求范围内，需要扩展（请求的开始日期在缓存内，但结束日期超出）
+    # 情况3: 缓存在请求范围内，需要扩展（请求的结束日期在缓存内，但开始日期超出）
+    # 情况4: 完全在范围内（无需获取）
+
+    if meta_time.end < need_time.begin or need_time.end < meta_time.begin:
+        remaining_range = TimeRange(begin=need_time.begin, end=need_time.end)
+        meta_time.begin = need_time.begin
+        meta_time.end = need_time.end
+        return remaining_range, meta_time
+    elif meta_time.begin < need_time.begin <= meta_time.end < need_time.end:
+        remaining_range = TimeRange(begin=meta_time.end, end=need_time.end)
+        meta_time.end = need_time.end
+        return remaining_range, meta_time
+    elif need_time.begin < meta_time.begin <= need_time.end < meta_time.end:
+        remaining_range = TimeRange(begin=need_time.begin, end=meta_time.begin)
+        meta_time.begin = need_time.begin
+        return remaining_range, meta_time
+    else:
+        return None, meta_time

wxmp-1.0.0/src/wxmp/tools/__init__.py

@@ -0,0 +1,20 @@
+from .article import generate_yaml_front_matter, save_article_content, save_markdown
+from .file import (
+    load_html,
+    load_json,
+    load_text,
+    sanitize_filename,
+    save_html,
+    save_json,
+    save_text,
+)
+
+__all__ = [
+    "generate_yaml_front_matter",
+    "save_article_content",
+    "save_markdown",
+    "load_html",
+    "load_text",
+    "save_html",
+    "save_text",
+]

wxmp-1.0.0/src/wxmp/tools/article.py

@@ -0,0 +1,199 @@
+import re
+from pathlib import Path
+from typing import Literal
+
+from .file import save_html, save_markdown
+
+
+def generate_yaml_front_matter(
+    title: str = "",
+    date_str: str = "",
+    link: str = "",
+    account_name: str = "",
+    digest: str = "",
+) -> str:
+    """
+    生成 YAML front matter 格式的元数据
+
+    Args:
+        title: 文章标题
+        date_str: 日期字符串
+        link: 文章链接
+        account_name: 公众号名称
+        digest: 文章摘要
+
+    Returns:
+        YAML front matter 字符串
+    """
+    yaml_front_matter = "---\n"
+    if title:
+        yaml_front_matter += f"title: {title}\n"
+    if date_str:
+        yaml_front_matter += f"date: {date_str}\n"
+    if link:
+        yaml_front_matter += f"link: {link}\n"
+    if account_name:
+        yaml_front_matter += f"account: {account_name}\n"
+    if digest:
+        yaml_front_matter += f"summary: {digest}\n"
+    yaml_front_matter += "---\n"
+    return yaml_front_matter
+
+
+def save_article_content(
+    html: str,
+    save_path: Path,
+    save_file: Literal["md", "html"] = "md",
+    title: str = "",
+    date_str: str = "",
+    link: str = "",
+    account_name: str = "",
+    digest: str = "",
+    min_file_size_kb: int = 3,
+) -> bool:
+    """
+    保存文章内容到文件（带元数据和文件大小检查）
+
+    Args:
+        html: 文章HTML内容
+        save_path: 保存路径
+        save_file: 保存格式（md 或 html）
+        title: 文章标题
+        date_str: 日期字符串
+        link: 文章链接
+        account_name: 公众号名称
+        digest: 文章摘要
+        min_file_size_kb: 最小文件大小（KB），小于此值会被删除
+
+    Returns:
+        是否成功保存
+    """
+    save_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if save_path.exists():
+        return True
+
+    try:
+        if save_file == "md":
+            yaml_front_matter = generate_yaml_front_matter(
+                title=title,
+                date_str=date_str,
+                link=link,
+                account_name=account_name,
+                digest=digest,
+            )
+
+            main_content = ""
+            content_match = re.search(
+                r'<div[^>]*id="js_content"[^>]*>(.*?)</div>', html, re.DOTALL
+            )
+
+            if content_match:
+                main_content = content_match.group(1)
+            else:
+                body_match = re.search(r"<body[^>]*>(.*?)</body>", html, re.DOTALL)
+                main_content = body_match.group(1) if body_match else html
+
+            markdown_content = yaml_front_matter + html_to_markdown(main_content)
+            save_markdown(markdown_content, save_path)
+        elif save_file == "html":
+            save_html(html, save_path)
+
+        file_size = save_path.stat().st_size
+        min_file_size_bytes = min_file_size_kb * 1024
+
+        if file_size < min_file_size_bytes:
+            save_path.unlink()
+            return False
+
+        return True
+
+    except Exception as e:
+        if save_path.exists():
+            try:
+                save_path.unlink()
+            except:
+                pass
+        return False
+
+
+def html_to_markdown(html: str) -> str:
+    """
+    Simple Regex-based HTML to Markdown converter.
+    """
+    # Remove style and script
+    html = re.sub(r"<style.*?>.*?</style>", "", html, flags=re.DOTALL)
+    html = re.sub(r"<script.*?>.*?</script>", "", html, flags=re.DOTALL)
+
+    # Extract images: <img ... data-src="..."> or <img ... src="...">
+    # Do this BEFORE removing any tags
+    def replace_img(match):
+        src = match.group(1) or match.group(2)
+        return f"\n![]({src})\n"
+
+    # Replace img tags with markdown images
+    html = re.sub(r'<img[^>]+data-src="([^"]+)"[^>]*>', replace_img, html)
+    html = re.sub(r'<img[^>]+src="([^"]+)"[^>]*>', replace_img, html)
+
+    # Handle code blocks - <pre><code>...</code></pre> or <pre>...</pre>
+    def replace_pre_code(match):
+        code_content = match.group(1)
+        # Remove inner <code> tags if present
+        code_content = re.sub(
+            r"<code[^>]*>(.*?)</code>", r"\1", code_content, flags=re.DOTALL
+        )
+        # Decode HTML entities in code
+        code_content = (
+            code_content.replace("&lt;", "<")
+            .replace("&gt;", ">")
+            .replace("&amp;", "&")
+            .replace("&quot;", '"')
+        )
+        code_content = code_content.replace("&nbsp;", " ")
+        return f"\n```\n{code_content}\n```\n"
+
+    html = re.sub(r"<pre[^>]*>(.*?)</pre>", replace_pre_code, html, flags=re.DOTALL)
+
+    # Handle inline code - <code>...</code>
+    html = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", html, flags=re.DOTALL)
+
+    # Remove lines that only contain HTML attributes (common in WeChat articles)
+    html = re.sub(
+        r"^\s*(class|data-|style|width|height|type|from|wx_fmt|data-ratio|data-type|data-w|data-imgfileid|data-aistatus|data-s)=[^>]*>\s*$",
+        "",
+        html,
+        flags=re.MULTILINE,
+    )
+
+    # Headers
+    for i in range(6, 0, -1):
+        html = re.sub(f"<h{i}[^>]*>(.*?)</h{i}>", "#" * i + r" \1\n", html)
+
+    # Paragraphs and Breaks
+    html = re.sub(r"<p[^>]*>", "\n", html)
+    html = re.sub(r"</p>", "\n", html)
+    html = re.sub(r"<br\s*/?>", "\n", html)
+
+    # Bold/Strong
+    html = re.sub(r"<(b|strong)[^>]*>(.*?)</\1>", r"**\2**", html)
+
+    # Lists (Simple)
+    html = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1\n", html)
+
+    # Remove all remaining tags (including self-closing)
+    html = re.sub(r"<[^>]+>", "", html)
+
+    # Decode entities (basic)
+    html = (
+        html.replace("&nbsp;", " ")
+        .replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&amp;", "&")
+        .replace("&quot;", '"')
+    )
+
+    # Collapse multiple newlines and spaces
+    html = re.sub(r"\n{3,}", "\n\n", html)
+    html = re.sub(r" +", " ", html)
+
+    return html.strip()

wxmp-1.0.0/src/wxmp/tools/file.py

@@ -0,0 +1,86 @@
+import json
+import re
+from pathlib import Path
+from typing import Union
+
+
+def sanitize_filename(filename: str, max_length: int = 200) -> str:
+    """
+    清理文件名，移除非法字符
+
+    Args:
+        filename: 原始文件名
+        max_length: 最大文件名长度
+
+    Returns:
+        清理后的合法文件名
+    """
+    # Windows 不允许的字符: \ / : * ? " < > |
+    illegal_chars = r'[\\/:*?"<>|]'
+
+    # 移除非法字符
+    filename = re.sub(illegal_chars, "_", filename)
+
+    # 移除控制字符
+    filename = "".join(char for char in filename if ord(char) >= 32)
+
+    # 移除首尾空格和点
+    filename = filename.strip(". ")
+
+    # 限制文件名长度
+    if len(filename) > max_length:
+        filename = filename[:max_length]
+
+    # 如果文件名为空，使用默认名称
+    if not filename:
+        filename = "untitled"
+
+    return filename
+
+
+def load_json(file_path: Union[str, Path]) -> dict:
+    file_path = Path(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_json(data: dict, file_path: Union[str, Path]):
+    file_path = Path(file_path)
+    with open(file_path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+
+
+def load_html(file_path: Union[str, Path]) -> str:
+    file_path = Path(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def save_html(html: str, file_path: Union[str, Path]):
+    file_path = Path(file_path)
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(html)
+
+
+def load_text(file_path: Union[str, Path]) -> str:
+    file_path = Path(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def save_text(text: str, file_path: Union[str, Path]):
+    file_path = Path(file_path)
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(text)
+
+
+def load_markdown(file_path: Union[str, Path]) -> str:
+    file_path = Path(file_path)
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def save_markdown(markdown: str, file_path: Union[str, Path]):
+    file_path = Path(file_path)
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(markdown)