wechat-article-parser 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/PKG-INFO +39 -1
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/README.md +38 -0
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/pyproject.toml +1 -1
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/parser.py +26 -5
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/tests/conftest.py +6 -0
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/tests/test_parser.py +10 -10
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/.gitignore +0 -0
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/LICENSE +0 -0
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/__init__.py +0 -0
- {wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/models.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wechat-article-parser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -75,12 +75,14 @@ print(result.article_markdown)
|
|
|
75
75
|
|
|
76
76
|
- `timeout`:请求超时时间,单位秒,默认 15
|
|
77
77
|
- `user_agent`:自定义 User-Agent,不传则使用内置默认值
|
|
78
|
+
- `proxy`:HTTP/HTTPS 代理地址,不传则直连
|
|
78
79
|
|
|
79
80
|
```python
|
|
80
81
|
result = parse(
|
|
81
82
|
"https://mp.weixin.qq.com/s/xxxxx",
|
|
82
83
|
timeout=30,
|
|
83
84
|
user_agent="MyBot/1.0",
|
|
85
|
+
proxy="http://user:pass@127.0.0.1:7890",
|
|
84
86
|
)
|
|
85
87
|
```
|
|
86
88
|
|
|
@@ -96,6 +98,7 @@ result = parse(
|
|
|
96
98
|
| `mp_alias` | `str` | 公众号别名 |
|
|
97
99
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
98
100
|
| `mp_description` | `str` | 公众号简介 |
|
|
101
|
+
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
99
102
|
| `article_id` | `str` | 文章 ID |
|
|
100
103
|
| `article_msg_id` | `int` | 文章所在的群发消息 ID |
|
|
101
104
|
| `article_idx` | `int` | 群发图文中的位置(从 1 开始) |
|
|
@@ -110,6 +113,27 @@ result = parse(
|
|
|
110
113
|
|
|
111
114
|
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_markdown`、`article_publish_time` 均不为空/零。
|
|
112
115
|
|
|
116
|
+
### 判断账号类型
|
|
117
|
+
|
|
118
|
+
`AccountType` 继承自 `str` 枚举,既支持枚举比较,也支持与中文字符串直接比较:
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from wechat_article_parser import parse, AccountType
|
|
122
|
+
|
|
123
|
+
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
124
|
+
|
|
125
|
+
# 推荐:枚举比较(有类型提示与 IDE 补全)
|
|
126
|
+
if result.mp_account_type == AccountType.SERVICE:
|
|
127
|
+
print("这是服务号")
|
|
128
|
+
|
|
129
|
+
# 也支持:字符串字面量比较
|
|
130
|
+
if result.mp_account_type == "服务号":
|
|
131
|
+
print("这是服务号")
|
|
132
|
+
|
|
133
|
+
# 打印直接输出中文值
|
|
134
|
+
print(f"账号类型: {result.mp_account_type}") # 账号类型: 订阅号
|
|
135
|
+
```
|
|
136
|
+
|
|
113
137
|
## 异常处理
|
|
114
138
|
|
|
115
139
|
### WeChatVerifyError
|
|
@@ -181,3 +205,17 @@ pytest tests/test_parser.py::test_fetch_all -s --url "https://mp.weixin.qq.com/s
|
|
|
181
205
|
```bash
|
|
182
206
|
pytest tests/test_parser.py::test_fetch_markdown -s --url "https://mp.weixin.qq.com/s/xxxxx"
|
|
183
207
|
```
|
|
208
|
+
|
|
209
|
+
### 通过 HTTP 代理运行测试
|
|
210
|
+
|
|
211
|
+
所有测试命令都支持 `--proxy` 参数,传入后所有请求都会经由该代理;不传则直连。常用于 IP 被微信限流时换出口:
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
# 全量测试走代理
|
|
215
|
+
pytest tests/test_parser.py -v -s --proxy "http://127.0.0.1:7890"
|
|
216
|
+
|
|
217
|
+
# 测试单个链接走代理
|
|
218
|
+
pytest tests/test_parser.py::test_fetch_all -s \
|
|
219
|
+
--url "https://mp.weixin.qq.com/s/xxxxx" \
|
|
220
|
+
--proxy "http://user:pass@127.0.0.1:7890"
|
|
221
|
+
```
|
|
@@ -60,12 +60,14 @@ print(result.article_markdown)
|
|
|
60
60
|
|
|
61
61
|
- `timeout`:请求超时时间,单位秒,默认 15
|
|
62
62
|
- `user_agent`:自定义 User-Agent,不传则使用内置默认值
|
|
63
|
+
- `proxy`:HTTP/HTTPS 代理地址,不传则直连
|
|
63
64
|
|
|
64
65
|
```python
|
|
65
66
|
result = parse(
|
|
66
67
|
"https://mp.weixin.qq.com/s/xxxxx",
|
|
67
68
|
timeout=30,
|
|
68
69
|
user_agent="MyBot/1.0",
|
|
70
|
+
proxy="http://user:pass@127.0.0.1:7890",
|
|
69
71
|
)
|
|
70
72
|
```
|
|
71
73
|
|
|
@@ -81,6 +83,7 @@ result = parse(
|
|
|
81
83
|
| `mp_alias` | `str` | 公众号别名 |
|
|
82
84
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
83
85
|
| `mp_description` | `str` | 公众号简介 |
|
|
86
|
+
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
84
87
|
| `article_id` | `str` | 文章 ID |
|
|
85
88
|
| `article_msg_id` | `int` | 文章所在的群发消息 ID |
|
|
86
89
|
| `article_idx` | `int` | 群发图文中的位置(从 1 开始) |
|
|
@@ -95,6 +98,27 @@ result = parse(
|
|
|
95
98
|
|
|
96
99
|
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_markdown`、`article_publish_time` 均不为空/零。
|
|
97
100
|
|
|
101
|
+
### 判断账号类型
|
|
102
|
+
|
|
103
|
+
`AccountType` 继承自 `str` 枚举,既支持枚举比较,也支持与中文字符串直接比较:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from wechat_article_parser import parse, AccountType
|
|
107
|
+
|
|
108
|
+
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
109
|
+
|
|
110
|
+
# 推荐:枚举比较(有类型提示与 IDE 补全)
|
|
111
|
+
if result.mp_account_type == AccountType.SERVICE:
|
|
112
|
+
print("这是服务号")
|
|
113
|
+
|
|
114
|
+
# 也支持:字符串字面量比较
|
|
115
|
+
if result.mp_account_type == "服务号":
|
|
116
|
+
print("这是服务号")
|
|
117
|
+
|
|
118
|
+
# 打印直接输出中文值
|
|
119
|
+
print(f"账号类型: {result.mp_account_type}") # 账号类型: 订阅号
|
|
120
|
+
```
|
|
121
|
+
|
|
98
122
|
## 异常处理
|
|
99
123
|
|
|
100
124
|
### WeChatVerifyError
|
|
@@ -166,3 +190,17 @@ pytest tests/test_parser.py::test_fetch_all -s --url "https://mp.weixin.qq.com/s
|
|
|
166
190
|
```bash
|
|
167
191
|
pytest tests/test_parser.py::test_fetch_markdown -s --url "https://mp.weixin.qq.com/s/xxxxx"
|
|
168
192
|
```
|
|
193
|
+
|
|
194
|
+
### 通过 HTTP 代理运行测试
|
|
195
|
+
|
|
196
|
+
所有测试命令都支持 `--proxy` 参数,传入后所有请求都会经由该代理;不传则直连。常用于 IP 被微信限流时换出口:
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
# 全量测试走代理
|
|
200
|
+
pytest tests/test_parser.py -v -s --proxy "http://127.0.0.1:7890"
|
|
201
|
+
|
|
202
|
+
# 测试单个链接走代理
|
|
203
|
+
pytest tests/test_parser.py::test_fetch_all -s \
|
|
204
|
+
--url "https://mp.weixin.qq.com/s/xxxxx" \
|
|
205
|
+
--proxy "http://user:pass@127.0.0.1:7890"
|
|
206
|
+
```
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wechat-article-parser"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.4"
|
|
8
8
|
description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
{wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/parser.py
RENAMED
|
@@ -476,30 +476,50 @@ def _parse_html(url: str, html: str) -> ArticleResult:
|
|
|
476
476
|
# 公开接口
|
|
477
477
|
# ---------------------------------------------------------------------------
|
|
478
478
|
|
|
479
|
-
def parse(
|
|
479
|
+
def parse(
|
|
480
|
+
url: str,
|
|
481
|
+
*,
|
|
482
|
+
timeout: int = _TIMEOUT,
|
|
483
|
+
user_agent: str | None = None,
|
|
484
|
+
proxy: str | None = None,
|
|
485
|
+
) -> ArticleResult:
|
|
480
486
|
"""抓取并解析微信公众号文章(同步方式)。
|
|
481
487
|
|
|
482
488
|
Args:
|
|
483
489
|
url: 微信文章链接。
|
|
484
490
|
timeout: 请求超时时间(秒)。
|
|
485
491
|
user_agent: 自定义 User-Agent,不传则使用内置默认值。
|
|
492
|
+
proxy: HTTP/HTTPS 代理地址,例如 "http://user:pass@host:port",不传则直连。
|
|
486
493
|
|
|
487
494
|
Returns:
|
|
488
495
|
包含解析数据的 ArticleResult。
|
|
489
496
|
"""
|
|
490
497
|
ua = user_agent or _USER_AGENT
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
498
|
+
with httpx.Client(
|
|
499
|
+
headers={"User-Agent": ua},
|
|
500
|
+
timeout=timeout,
|
|
501
|
+
follow_redirects=True,
|
|
502
|
+
proxy=proxy,
|
|
503
|
+
) as client:
|
|
504
|
+
response = client.get(url)
|
|
505
|
+
response.raise_for_status()
|
|
506
|
+
return _parse_html(url, response.text)
|
|
494
507
|
|
|
495
508
|
|
|
496
|
-
async def parse_async(
|
|
509
|
+
async def parse_async(
|
|
510
|
+
url: str,
|
|
511
|
+
*,
|
|
512
|
+
timeout: int = _TIMEOUT,
|
|
513
|
+
user_agent: str | None = None,
|
|
514
|
+
proxy: str | None = None,
|
|
515
|
+
) -> ArticleResult:
|
|
497
516
|
"""抓取并解析微信公众号文章(异步方式)。
|
|
498
517
|
|
|
499
518
|
Args:
|
|
500
519
|
url: 微信文章链接。
|
|
501
520
|
timeout: 请求超时时间(秒)。
|
|
502
521
|
user_agent: 自定义 User-Agent,不传则使用内置默认值。
|
|
522
|
+
proxy: HTTP/HTTPS 代理地址,例如 "http://user:pass@host:port",不传则直连。
|
|
503
523
|
|
|
504
524
|
Returns:
|
|
505
525
|
包含解析数据的 ArticleResult。
|
|
@@ -509,6 +529,7 @@ async def parse_async(url: str, *, timeout: int = _TIMEOUT, user_agent: str | No
|
|
|
509
529
|
headers={"User-Agent": ua},
|
|
510
530
|
timeout=timeout,
|
|
511
531
|
follow_redirects=True,
|
|
532
|
+
proxy=proxy,
|
|
512
533
|
) as client:
|
|
513
534
|
response = await client.get(url)
|
|
514
535
|
response.raise_for_status()
|
|
@@ -3,6 +3,7 @@ import pytest
|
|
|
3
3
|
|
|
4
4
|
def pytest_addoption(parser):
|
|
5
5
|
parser.addoption("--url", default=None, help="微信公众号文章链接")
|
|
6
|
+
parser.addoption("--proxy", default=None, help="HTTP 代理地址,例如 http://127.0.0.1:7890")
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
@pytest.fixture
|
|
@@ -11,3 +12,8 @@ def url(request):
|
|
|
11
12
|
if not value:
|
|
12
13
|
pytest.skip("需要通过 --url 参数传入链接")
|
|
13
14
|
return value
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def proxy(request):
|
|
19
|
+
return request.config.getoption("--proxy")
|
|
@@ -18,9 +18,9 @@ TEST_URLS = [
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@pytest.mark.parametrize("url", TEST_URLS)
|
|
21
|
-
def test_parse_sync(url: str) -> None:
|
|
21
|
+
def test_parse_sync(url: str, proxy: str | None) -> None:
|
|
22
22
|
try:
|
|
23
|
-
result = parse(url)
|
|
23
|
+
result = parse(url, proxy=proxy)
|
|
24
24
|
except WeChatVerifyError:
|
|
25
25
|
pytest.skip("WeChat returned verification page (IP rate-limited)")
|
|
26
26
|
return
|
|
@@ -29,9 +29,9 @@ def test_parse_sync(url: str) -> None:
|
|
|
29
29
|
|
|
30
30
|
@pytest.mark.parametrize("url", TEST_URLS)
|
|
31
31
|
@pytest.mark.asyncio
|
|
32
|
-
async def test_parse_async(url: str) -> None:
|
|
32
|
+
async def test_parse_async(url: str, proxy: str | None) -> None:
|
|
33
33
|
try:
|
|
34
|
-
result = await parse_async(url)
|
|
34
|
+
result = await parse_async(url, proxy=proxy)
|
|
35
35
|
except WeChatVerifyError:
|
|
36
36
|
pytest.skip("WeChat returned verification page (IP rate-limited)")
|
|
37
37
|
return
|
|
@@ -89,12 +89,12 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
89
89
|
assert result.is_valid
|
|
90
90
|
|
|
91
91
|
|
|
92
|
-
def test_fetch_all(url: str) -> None:
|
|
92
|
+
def test_fetch_all(url: str, proxy: str | None) -> None:
|
|
93
93
|
"""抓取指定 URL 并打印所有采集到的参数。
|
|
94
94
|
|
|
95
|
-
用法: pytest tests/test_parser.py::test_fetch_all -s --url <URL>
|
|
95
|
+
用法: pytest tests/test_parser.py::test_fetch_all -s --url <URL> [--proxy <PROXY>]
|
|
96
96
|
"""
|
|
97
|
-
result = parse(url)
|
|
97
|
+
result = parse(url, proxy=proxy)
|
|
98
98
|
print(f"\n{'='*60}")
|
|
99
99
|
print(f"URL: {url}")
|
|
100
100
|
print(f"公众号ID(B64):{result.mp_id_b64}")
|
|
@@ -120,10 +120,10 @@ def test_fetch_all(url: str) -> None:
|
|
|
120
120
|
print(f"{'='*60}")
|
|
121
121
|
|
|
122
122
|
|
|
123
|
-
def test_fetch_markdown(url: str) -> None:
|
|
123
|
+
def test_fetch_markdown(url: str, proxy: str | None) -> None:
|
|
124
124
|
"""抓取指定 URL 并只打印 Markdown 内容。
|
|
125
125
|
|
|
126
|
-
用法: pytest tests/test_parser.py::test_fetch_markdown -s --url <URL>
|
|
126
|
+
用法: pytest tests/test_parser.py::test_fetch_markdown -s --url <URL> [--proxy <PROXY>]
|
|
127
127
|
"""
|
|
128
|
-
result = parse(url)
|
|
128
|
+
result = parse(url, proxy=proxy)
|
|
129
129
|
print(f"\n{result.article_markdown}")
|
|
File without changes
|
|
File without changes
|
{wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/__init__.py
RENAMED
|
File without changes
|
{wechat_article_parser-0.0.3 → wechat_article_parser-0.0.4}/src/wechat_article_parser/models.py
RENAMED
|
File without changes
|