wechat-article-parser 0.0.7__tar.gz → 0.0.8__tar.gz

{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: wechat-article-parser
-Version: 0.0.7
+Version: 0.0.8
 Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
 License-Expression: MIT
 License-File: LICENSE

{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "wechat-article-parser"
-version = "0.0.7"
+version = "0.0.8"
 description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
 readme = "README.md"
 requires-python = ">=3.10"

{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.8}/src/wechat_article_parser/parser.py

@@ -35,12 +35,22 @@ _JS_DATA_TOKEN_RE = re.compile(
 # ---------------------------------------------------------------------------
 
 class _MarkdownConverter(MarkdownConverter):
-    def convert_span(self, el, text, parent_tags):
-        return text.strip()
+    def convert_br(self, el, text, parent_tags):
+        # 代码中的换行不需要 Markdown 硬换行所用的两个尾随空格。
+        if "pre" in parent_tags:
+            return "\n"
+        return super().convert_br(el, text, parent_tags)
 
-    def convert_p(self, el, text, parent_tags):
-        text = " ".join(text.split())
-        return super().convert_p(el, text, parent_tags)
+    def convert_pre(self, el, text, parent_tags):
+        # BeautifulSoup 已解码  ；代码中将其还原为可直接复制的普通空格。
+        return super().convert_pre(el, text.replace("\xa0", " "), parent_tags)
+
+    def convert_td(self, el, text, parent_tags):
+        # GFM 表格内的管道符即使位于行内代码中，也必须转义。
+        return super().convert_td(el, text.replace("|", r"\|"), parent_tags)
+
+    def convert_th(self, el, text, parent_tags):
+        return super().convert_th(el, text.replace("|", r"\|"), parent_tags)
 
 
 def _to_markdown(html: str) -> str:
@@ -50,8 +60,7 @@ def _to_markdown(html: str) -> str:
 def _to_markdown_plain(html: str) -> str:
     """用于纯文本类内容（纯文本/转载/视频/轮播/全屏）的转换。
 
-    这些内容由 <br> 承载换行，必须保留；故使用未覆写 convert_p 的基础转换器，
-    避免 _MarkdownConverter 折叠空白时把换行一并吞掉。
+    这些内容由 <br> 承载换行，使用基础转换器保留换行。
     """
     return MarkdownConverter().convert(html)
 
@@ -332,7 +341,8 @@ def _extract_swiper_meta(script_text: str, result: ArticleResult) -> None:
 
 def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None:
     """将 rich_media_content 区块转换为 Markdown。"""
-    soup = BeautifulSoup(content_tag.prettify(), "html.parser")
+    # prettify 会在行内标签之间插入空白，改变代码和引用链接的内容。
+    soup = BeautifulSoup(str(content_tag), "html.parser")
     seen: set[str] = set()
 
     # 处理 <img> 标签
@@ -385,7 +395,7 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
     for tag in to_remove:
         tag.decompose()
 
-    result.article_markdown = _to_markdown(soup.prettify())
+    result.article_markdown = _to_markdown(str(soup))
 
 
 def _extract_repost_content(content_tag: Tag, result: ArticleResult) -> None:

wechat_article_parser-0.0.8/tests/test_markdown.py

@@ -0,0 +1,85 @@
+"""富文本 Markdown 转换的离线回归测试。"""
+
+import pytest
+
+from wechat_article_parser.parser import _parse_html
+
+
+def _convert(body: str) -> str:
+    html = f'<div class="rich_media_content">{body}</div>'
+    return _parse_html("https://mp.weixin.qq.com/s/test", html).article_markdown
+
+
+def test_highlighted_code_preserves_breaks_spaces_and_entities() -> None:
+    # 微信 mdnice 代码块：语法高亮 span 切分 token，br 换行，nbsp 占位。
+    body = (
+        '<pre data-tool="mdnice编辑器"><code>'
+        '<span><span># ======== 基础体验 ========</span></span><span><br></span>'
+        '<span><span>set</span></span><span>&nbsp;-g default-terminal&nbsp;</span>'
+        '<span><span>"tmux-256color"</span></span><span><br></span>'
+        '<span>set</span><span>&nbsp;-as terminal-features&nbsp;</span>'
+        '<span>",*:RGB"</span><span><br></span>'
+        '<span>set</span><span>&nbsp;-g mouse on &nbsp; &nbsp;</span>'
+        '<span># 支持鼠标操作</span><span><br></span><span><br></span>'
+        '<span>&nbsp;&nbsp;echo &quot;&lt;tag&gt;&quot; &amp;&amp; echo &#160;ok</span>'
+        '</code></pre>'
+    )
+
+    assert _convert(body) == (
+        '```\n'
+        '# ======== 基础体验 ========\n'
+        'set -g default-terminal "tmux-256color"\n'
+        'set -as terminal-features ",*:RGB"\n'
+        'set -g mouse on    # 支持鼠标操作\n\n'
+        '  echo "<tag>" && echo  ok\n'
+        '```'
+    )
+
+
+def test_plain_pre_preserves_indentation_and_blank_lines() -> None:
+    assert _convert('<pre><code>if ready:\n    run()\n\n    done()\n</code></pre>') == (
+        '```\nif ready:\n    run()\n\n    done()\n```'
+    )
+
+
+@pytest.mark.parametrize("cell", ["td", "th"])
+def test_table_pipes_in_text_and_inline_code_are_escaped(cell: str) -> None:
+    output = _convert(
+        f'<table><tr><{cell}>A | B</{cell}><{cell}><section>'
+        '<span>验证 </span><code><span>tailscale status | grep peer-relay</span></code>'
+        f'</section></{cell}></tr></table>'
+    )
+
+    assert r'A \| B' in output
+    assert r'`tailscale status \| grep peer-relay`' in output
+    # 每行仍然只有两个单元格（三个未转义的分隔符）。
+    assert all(line.replace(r'\|', '').count('|') == 3 for line in output.splitlines())
+
+
+def test_pipe_outside_table_is_unchanged() -> None:
+    assert _convert('<p><code>left | right</code></p><pre>left | right</pre>') == (
+        '`left | right`\n\n```\nleft | right\n```'
+    )
+
+
+def test_inline_spans_and_paragraph_breaks_preserve_spacing() -> None:
+    assert _convert(
+        '<p><span>hello </span><span>world</span><br><span>next line</span></p>'
+    ) == 'hello world  \nnext line'
+
+
+def test_bold_reference_stays_inline() -> None:
+    assert _convert(
+        '<ul><li><section><strong><span>Hermes</span><sup>'
+        '<a href="https://example.com/hermes">[1]</a></sup></strong>'
+        '<span>：Nous Research 的自主 AI Agent</span></section></li></ul>'
+    ) == '* **Hermes[[1]](https://example.com/hermes)**：Nous Research 的自主 AI Agent'
+
+
+def test_literal_asterisks_in_source_stay_literal_and_inline() -> None:
+    # 这篇原文没有 strong 标签；不能把普通星号自动猜测为加粗语义。
+    assert _convert(
+        '<ul><li><section><span>**</span><span>Hermes</span>'
+        '<sup><span>[1]</span></sup><span>**：Nous Research 的自主 AI Agent</span>'
+        '</section></li></ul>'
+    ) == r'* \*\*Hermes[1]\*\*：Nous Research 的自主 AI Agent'