webpage-parser 0.1.0__py3-none-any.whl

webpage_parser/content.py

@@ -0,0 +1,915 @@
+"""content 抽取：两段式（调研03 §3.2）。
+
+阶段 A：DOM 预清理 + 图片白名单改写（_IMAGES 校验 -> 原站 URL -> 后续 img_ref 映射）
+阶段 B：trafilatura 2.2.0 fast markdown 抽取 + Markdown 后处理
+        （强/弱噪声双表 + 长度闸门、独立来源标注首段删除、空白归一）
+
+任何子步骤失败向外抛出由 pipeline 兜底为空串。
+"""
+import re
+from copy import deepcopy
+from urllib.parse import urlparse
+
+from lxml import html as lxml_html
+from lxml import etree
+
+try:
+    from .util import clean_text, norm_key, has_cjk
+    from .htmlprep import region_token_hit
+    from .timetext import parse_absolute
+except ImportError:
+    from util import clean_text, norm_key, has_cjk
+    from htmlprep import region_token_hit
+    from timetext import parse_absolute
+
+_REMOVE_TAGS = ("script", "style", "noscript", "template", "svg", "iframe",
+                "canvas", "form", "button", "input", "select", "textarea")
+_BOILER_TOKENS = {
+    "sidebar", "breadcrumb", "breadcrumbs", "related", "recommend", "recommended",
+    "footer", "copyright", "promo", "advert", "banner", "cookie", "subscribe",
+    "newsletter", "hot", "rank", "ranking", "location", "qrcode", "share",
+    # statement：网易系 post_statement/creative_statement 平台免责声明容器
+    #（与 post_body 并列、非正文；验收反馈 case_01/06/08/09）。token 精确匹配，
+    # "statements" 不命中。
+    "statement",
+    # nav/menu 误伤率高（如 module-nav 是数据表命名），单独做密度/长度门控
+}
+_BOILER_NAVY_TOKENS = {"nav", "menu"}
+_COMMENT_TOKENS = {"comment", "comments", "discuss", "reply"}
+_MODULE_TITLE_RE = re.compile(
+    r"^(.{0,10}(大家都在|也看过|猜你喜欢|相关推荐|延伸阅读|友情链接|合作伙伴).{0,4}|"
+    r"(相关|推荐|最新|热门)(文章|新闻|阅读|资讯|内容)|"
+    r".{1,10}分站|热门.{0,8}(推荐|排行|榜))$")
+_INVISIBLE_RE = re.compile(r"display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0",
+                           re.IGNORECASE)
+
+_IMG_ATTRS = ("src", "data-src", "data-original", "data-lazy-src", "data-actualsrc",
+              "data-url", "data-echo", "data-img")
+_QUERY_IGNORE = {"auth", "sig", "sign", "width", "height", "w", "h", "quality",
+                 "strip", "format", "smart", "fit", "auto", "token", "t", "size"}
+
+_STRONG_NOISE = {
+    "\u5c55\u5f00\u5168\u6587", "\u67e5\u770b\u539f\u6587", "\u9605\u8bfb\u5168\u6587",
+    "\u6536\u8d77", "\u76f8\u5173\u9605\u8bfb", "\u731c\u4f60\u559c\u6b22",
+    "\u70ed\u95e8\u63a8\u8350", "\u4e3a\u4f60\u63a8\u8350", "\u5ef6\u4f38\u9605\u8bfb",
+    "\u5927\u5bb6\u90fd\u5728\u770b", "\u4e0a\u4e00\u9875", "\u4e0b\u4e00\u9875",
+    "\u5206\u4eab\u5230", "\u6536\u85cf", "\u70b9\u8d5e", "\u8f6c\u53d1", "\u4e3e\u62a5",
+    "\u6211\u6765\u8bf4\u4e24\u53e5", "\u53d1\u8868\u4f60\u7684\u770b\u6cd5",
+    "\u67e5\u770b\u5168\u90e8\u8bc4\u8bba", "\u6682\u65e0\u8bc4\u8bba",
+    "\u9996\u9875", "\u767b\u5f55", "\u6ce8\u518c", "\u66f4\u591a", "\u641c\u7d22",
+    "\u8fd4\u56de\u9876\u90e8", "\u70b9\u51fb\u67e5\u770b", "\u67e5\u770b\u66f4\u591a",
+    "\u6211\u8981\u8bc4\u8bba", "\u4e3e\u62a5/\u53cd\u9988", "\u5206\u4eab",
+    "\u624b\u673a\u770b", "Contact Us",
+}
+_WEAK_NOISE = (
+    "\u767b\u5f55\u540e\u624d\u80fd", "\u4e0b\u8f7d\u5ba2\u6237\u7aef",
+    "\u6253\u5f00\u5c0f\u7a0b\u5e8f", "\u626b\u7801\u67e5\u770b", "\u7248\u6743\u6240\u6709",
+    "\u672a\u7ecf\u6388\u6743", "\u4e0d\u5f97\u8f6c\u8f7d", "all rights reserved",
+    "icp\u5907\u6848", "\u4eac\u516c\u7f51\u5b89\u5907", "cookie", "\u9690\u79c1",
+    "\u9a8c\u8bc1\u7801", "\u8bbf\u95ee\u8fc7\u4e8e\u9891\u7e41", "\u5173\u6ce8\u516c\u4f17\u53f7",
+    "\u6253\u5f00app", "\u5728app\u5185\u6253\u5f00", "\u4e00\u952e\u767b\u5f55",
+    "\u672c\u7ad9\u4f7f\u7528cookie", "skip to content", "skip to main",
+    "preferred source",
+)
+_TECH_RE = re.compile(r"^(function\s*\(|var\s+\w+\s*=|@media|\./tpl_|const\s+\w+\s*=)")
+_SRC_LABEL_RE = re.compile(
+    r"^\s*(\u6765\u6e90|\u6765\u81ea|\u51fa\u5904|\u6587\u7ae0\u6765\u6e90|\u7a3f\u6e90|"
+    r"\u7f16\u8f91|\u8d23\u7f16|\u8d23\u4efb\u7f16\u8f91|\u4f5c\u8005)\s*[:：]")
+
+_MD_IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)\s]+)[^)]*\)")
+# 关注引导短行（"点击上方 蓝字 关注，获取更多资讯"类；验收反馈 case_01）
+_FOLLOW_GUIDE_RE = re.compile(
+    r"^(?:\u70b9\u51fb|\u957f\u6309|\u626b\u63cf|\u626b\u7801).{0,12}?"
+    r"(?:\u5173\u6ce8|\u8ba2\u9605)")
+# App 导流短行（"更多热点速报…尽在北京日报App"，验收反馈 case_03）
+_APP_GUIDE_RE = re.compile(
+    r"(?:\u5c3d\u5728|\u5c31\u5728|\u626b\u7801|\u70b9\u51fb|\u4e0b\u8f7d|\u6253\u5f00)"
+    r".{0,24}(?:App|APP|app)\s*$")
+# 英文付费墙/订阅 CTA 短行（Bloomberg 等，验收反馈 case_04）
+_EN_CTA_RE = re.compile(
+    r"already a subscriber|log\s*in to (?:keep reading|continue)|"
+    r"(?:subscribe|sign up) to (?:continue|keep)|to continue reading",
+    re.IGNORECASE)
+# 仅有标签词 + 冒号的悬挂行（"Chủ đề:" / "话题：" / "Tags:"，验收反馈 case_01）
+_LABEL_ONLY_RE = re.compile(
+    r"^(?:\u8bdd\u9898|\u6807\u7b7e|\u5206\u7c7b|\u4e3b\u9898|"
+    r"ch\u1ee7\s*\u0111\u1ec1|tags?|categor(?:y|ies)|topics?|keywords?)"
+    r"\s*[:\uff1a]\s*$",
+    re.IGNORECASE)
+
+# ---- GBK/UTF-8 双重编码乱码检测（验收反馈：乱码块必须丢弃，不得原样倒出）----
+_PUA_RE = re.compile(r"[\ue000-\uf8ff]")
+_MOJI_EURO_RE = re.compile(r"[\u4e00-\u9fff]\u20ac|\u20ac[\u4e00-\u9fff]")
+# GBK->UTF-8 乱码中的高频罕见字（从真实乱码块统计字符频率，再减去同页正常
+# 正文区出现的字与 plausible 用字后得到；正常中文不会成簇出现这些字）
+_MOJI_CHARS = ("\u6924\u572d\u6d30\u6d7c\u95ab\u934f\u90f4\u5f37\u9352\u9354\u93c8\u59df"
+               "\u8bf2\u5d07\u53e7\u7eef\u7d1d\u59e9\u9429\u4f78\u6c2c\u52eb\u61b3\u9365"
+               "\u69e6\u5bee\u6e47\u7b1f\u6b91\u4f77\u951b\u93ac\u93c3\u74ba\u6d93\u93bb"
+               "\u935b\u6a3b\u5d85\u5a23\u59de\u6769\u752f\u59ea\u93ae\u9473\u9286\u74d1"
+               "\u4f79\u95b2\u9366\u677f\u7aa1\u9471\u6c2b\u7d30\u93c2\u4f34\u6427\u9368"
+               "\u5b2c\u7ecb\u5b2a\u7c2d\u6a3a\u4f10\u8702\u7d12\u6c31\u608a\u7490\u5f7f"
+               "\u546d\u5978\u6336\u9356\u5474\u95c8\u6540\u8930\u66df\u9367\u6943\u6ae5"
+               "\u655e\u9350")
+_MOJI_SET = frozenset(_MOJI_CHARS)
+
+
+def mojibake_score(text):
+    if not text:
+        return 0
+    s = len(_PUA_RE.findall(text)) * 2
+    s += text.count("\ufffd")
+    s += len(_MOJI_EURO_RE.findall(text)) * 2
+    s += sum(1 for c in text if c in _MOJI_SET)
+    return s
+
+
+# ---- 推广/营销区块（验收反馈 b：含推广特征词的区块做负样本过滤）----
+_PROMO_STRONG_RE = re.compile(
+    r"\u7b7e\u5355\u603b\u989d|\u53ef\u80fd\u8ba4\u8bc6\u7684\u597d\u53cb|"
+    r"\u626b\u7801\u5b89\u88c5|\u5f00\u901a\u4f1a\u5458\u670d\u52a1|"
+    r"\u4eba\u8109\u5708|\u50ac\u4fc3\u66f4\u65b0|\u52a0\u5165\u5929\u5de5"
+)
+_PROMO_WEAK_RE = re.compile(
+    r"\u5f00\u901a\u4f1a\u5458|\u4e0b\u8f7d\s*[Aa][Pp][Pp]|\u6253\u5f00\s*[Aa][Pp][Pp]|"
+    r"\u514d\u8d39\u9886\u53d6|\u4f18\u60e0\u5238|\u4f1a\u5458\u4e13\u4eab|"
+    r"\u5c0a\u4eab|VIP\u4f1a\u5458|\u7acb\u5373\u5f00\u901a|\u7acb\u5373\u4e0b\u8f7d|"
+    r"\u70b9\u51fb\u4e0b\u8f7d|\u626b\u7801\u4e0b\u8f7d|\u626b\u7801\u5173\u6ce8|"
+    r"\u5173\u6ce8\u516c\u4f17\u53f7|\u9650\u65f6\u4f18\u60e0|\u7acb\u51cf"
+)
+_POPUP_TOKENS = {"pop", "popup", "modal", "dialog", "mask", "layer", "float",
+                 "fixed", "suspension", "hoverbox"}
+
+# data-test/testid 的 CTA 容器（Bloomberg article-cta 等，验收反馈 case_04）
+_CTA_TESTID_RE = re.compile(r"cta|paywall|subscribe|login|signup|sign-up")
+# 版权/免责声明块（验收反馈 case_02 新浪红字声明）
+_DECLARATION_RE = re.compile(r"^(特别声明|免责声明)\s*[:：]?")
+# AI 生成声明（验收反馈 case_02 "本文由AI生成"）
+_AI_GEN_RE = re.compile(r"^本文由.{0,4}AI.{0,4}生成")
+
+
+def _promo_hits(text):
+    return len(set(_PROMO_WEAK_RE.findall(text)))
+
+
+_traf = None
+_traf_cfg = None
+
+
+def _get_trafilatura():
+    global _traf, _traf_cfg
+    if _traf is None:
+        import trafilatura
+        from trafilatura.settings import use_config
+        cfg = use_config()
+        cfg.set("DEFAULT", "MIN_EXTRACTED_SIZE", "80")
+        cfg.set("DEFAULT", "MIN_OUTPUT_SIZE", "1")
+        cfg.set("DEFAULT", "MIN_OUTPUT_COMM", "1")
+        _traf = trafilatura
+        _traf_cfg = cfg
+    return _traf, _traf_cfg
+
+
+def _norm_img_url(u):
+    """归一化图片 URL：去 scheme、去签名/尺寸类 query。"""
+    try:
+        p = urlparse(u.strip())
+    except Exception:
+        return ""
+    host = (p.hostname or "").lower()
+    path = p.path or ""
+    if not host or not path:
+        return ""
+    from urllib.parse import parse_qsl
+    kept = [(k, v) for k, v in parse_qsl(p.query) if k.lower() not in _QUERY_IGNORE]
+    kept.sort()
+    q = "&".join(f"{k}={v}" for k, v in kept)
+    return host + path + ("?" + q if q else "")
+
+
+def _img_basename(u):
+    try:
+        path = urlparse(u.strip()).path
+    except Exception:
+        return ""
+    return path.rsplit("/", 1)[-1].lower()
+
+
+_SIZE_SUFFIX_RE = re.compile(r"-(?:\d{2,4}x\d{2,4}|scaled|large|medium|small|thumbnail)(?=\.[a-z]{3,4}$)")
+
+
+def _img_base_stripped(u):
+    """去掉 WP 式尺寸后缀后的 basename：LAKEVIEW-2-634x312.jpg -> lakeview-2.jpg。"""
+    b = _img_basename(u)
+    return _SIZE_SUFFIX_RE.sub("", b)
+
+
+class _ImageMap:
+    def __init__(self, images, img_ref):
+        self.exact = {}
+        self.norm = {}
+        self.path = {}
+        self.base = {}
+        self.base_stripped = {}
+        self.allowed = bool(images)
+        for u in images or []:
+            if not isinstance(u, str) or not u:
+                continue
+            self.exact[u] = u
+            self.norm.setdefault(_norm_img_url(u), u)
+            nk = _norm_img_url(u)
+            if nk:
+                self.path.setdefault(nk.split("?", 1)[0], u)
+            self.base.setdefault(_img_basename(u), u)
+            self.base_stripped.setdefault(_img_base_stripped(u), u)
+        self.ref = img_ref if isinstance(img_ref, dict) else {}
+
+    def match(self, raw):
+        """返回 (白名单原站 URL, OSS URL) 或 None。raw 可为候选 list。"""
+        if not self.allowed or not raw:
+            return None
+        cands = [raw] if isinstance(raw, str) else [c for c in raw if c]
+        for cand in cands:
+            cand = cand.strip()
+            if not cand or cand.startswith("data:"):
+                continue
+            key = self.exact.get(cand)
+            if key is None:
+                key = self.norm.get(_norm_img_url(cand))
+            if key is None:
+                nk = _norm_img_url(cand)
+                if nk:
+                    key = self.path.get(nk.split("?", 1)[0])
+            if key is None:
+                key = self.base.get(_img_basename(cand))
+            if key is None:
+                key = self.base_stripped.get(_img_base_stripped(cand))
+            if key is None:
+                continue
+            oss = self.ref.get(key)
+            if oss:
+                return key, oss
+        return None
+
+
+def _preclean_tree(root, keep_comments):
+    for tag in _REMOVE_TAGS:
+        for el in root.iter(tag):
+            parent = el.getparent()
+            if parent is not None:
+                parent.remove(el)
+    for el in list(root.iter()):
+        if not isinstance(el.tag, str):
+            continue
+        style = el.get("style") or ""
+        if _INVISIBLE_RE.search(style) or el.get("hidden") is not None:
+            parent = el.getparent()
+            if parent is not None:
+                parent.remove(el)
+            continue
+        if (el.get("aria-hidden") or "").lower() == "true":
+            parent = el.getparent()
+            if parent is not None:
+                parent.remove(el)
+            continue
+    for el in list(root.iter()):
+        if not isinstance(el.tag, str):
+            continue
+        if el.tag in ("nav", "header", "footer", "aside"):
+            parent = el.getparent()
+            if parent is not None:
+                parent.remove(el)
+            continue
+        role = el.get("role") or ""
+        if role in ("navigation", "banner", "complementary", "search"):
+            parent = el.getparent()
+            if parent is not None:
+                parent.remove(el)
+            continue
+        if el.tag in ("div", "section", "ul", "span"):
+            attr = ((el.get("class") or "") + " " + (el.get("id") or "")).lower()
+            hit_boiler = region_token_hit(attr, _BOILER_TOKENS)
+            hit_navy = region_token_hit(attr, _BOILER_NAVY_TOKENS)
+            if (hit_boiler or hit_navy) and not (
+                    keep_comments and region_token_hit(attr, _COMMENT_TOKENS)):
+                # 大容器豁免：页面级布局包装器常带 banner/hot 等 token
+                # （dailymail class="page-banner articlepage" 包着全部正文），
+                # 含 h1 或 >4000 字的容器删了就是灾难，跳过
+                if el.findall(".//h1"):
+                    continue
+                try:
+                    _el_txt = clean_text(el.text_content())
+                except Exception:
+                    continue
+                if len(_el_txt) > 4000:
+                    continue
+                remove = True
+                if hit_navy and not hit_boiler:
+                    # 纯 nav/menu token：只对链接密集或短文本块下手
+                    try:
+                        txt = _el_txt
+                        atxt = "".join(clean_text(a.text_content())
+                                       for a in el.iter("a"))
+                        remove = len(txt) < 200 or len(atxt) > 0.5 * max(len(txt), 1)
+                    except Exception:
+                        remove = False
+                if remove:
+                    parent = el.getparent()
+                    if parent is not None:
+                        parent.remove(el)
+    # CTA / 声明 / 平台引导区块（验收反馈 case_02/03/04）
+    for el in list(root.iter()):
+        if not isinstance(el.tag, str):
+            continue
+        parent = el.getparent()
+        if parent is None:
+            continue
+        datatest = ((el.get("data-test") or "") + " " +
+                    (el.get("data-testid") or "")).lower()
+        if datatest and _CTA_TESTID_RE.search(datatest):
+            parent.remove(el)
+            continue
+        cls_id = ((el.get("class") or "") + " " + (el.get("id") or "")).lower()
+        if el.get("data-bjh-src-hide") is not None or "bjh-a-hide" in cls_id:
+            parent.remove(el)
+            continue
+        if el.tag not in ("div", "p", "section", "aside", "span"):
+            continue
+        if el.findall(".//h1"):
+            continue
+        try:
+            t = clean_text(el.text_content())
+        except Exception:
+            continue
+        if not t or len(t) > 400:
+            continue
+        if _DECLARATION_RE.match(t):
+            parent.remove(el)
+            continue
+        if len(t) <= 60 and _AI_GEN_RE.search(t):
+            parent.remove(el)
+            continue
+    # 模块标题板：短标题命中站点样板词 → 删所属模块（大家都在问/天工网分站/猜你喜欢…）
+    for el in list(root.iter()):
+        if el.tag not in ("h2", "h3", "h4", "div", "p", "span"):
+            continue
+        parent = el.getparent()
+        if parent is None:
+            continue
+        try:
+            t = clean_text(el.text_content())
+        except Exception:
+            continue
+        if not t or len(t) > 14 or not _MODULE_TITLE_RE.match(t):
+            continue
+        try:
+            ptxt = clean_text(parent.text_content())
+        except Exception:
+            continue
+        if len(ptxt) > 4000 or parent.findall(".//h1"):
+            continue
+        gp = parent.getparent()
+        if gp is not None and parent.getparent() is not None:
+            gp.remove(parent)
+    # 弹窗 / 推广 / 乱码块：最深优先，含 h1 或超长文本的容器不删（验收反馈 b/c/d）
+    for el in reversed(list(root.iter())):
+        if not isinstance(el.tag, str):
+            continue
+        if el.tag not in ("div", "section", "aside", "ul", "table"):
+            continue
+        parent = el.getparent()
+        if parent is None:
+            continue
+        try:
+            text = clean_text(el.text_content())
+        except Exception:
+            continue
+        tl = len(text)
+        if tl < 2 or tl > 4000:
+            continue
+        if el.findall(".//h1"):
+            continue
+        cid = ((el.get("class") or "") + " " + (el.get("id") or "")).lower()
+        if region_token_hit(cid, _POPUP_TOKENS) and tl < 2000:
+            parent.remove(el)
+            continue
+        if _PROMO_STRONG_RE.search(text):
+            parent.remove(el)
+            continue
+        if _promo_hits(text) >= 2 and tl < 1500:
+            parent.remove(el)
+            continue
+        if _mojibake_heavy(text):
+            parent.remove(el)
+            continue
+    return root
+
+
+def _mojibake_heavy(text):
+    """局部乱码块判定：只看密度（PUA 图标字体/长文罕见字积累不触发）。"""
+    sc = mojibake_score(text)
+    if sc < 3:
+        return False
+    cjk = sum(1 for c in text if "一" <= c <= "鿿")
+    if cjk == 0:
+        return sc >= 6
+    return sc * 2 >= cjk
+
+
+_IMG_HREF_RE = re.compile(r"\.(?:jpe?g|png|gif|webp|bmp)(?:[?#]|$)", re.IGNORECASE)
+
+
+def _img_candidates(img):
+    cands = []
+    for attr in _IMG_ATTRS:
+        v = img.get(attr)
+        if v and v.strip():
+            cands.append(v.strip())
+    srcset = img.get("srcset") or ""
+    for part in srcset.split(","):
+        u = part.strip().split(" ")[0]
+        if u:
+            cands.append(u)
+    parent = img.getparent()
+    if parent is not None and parent.tag == "a":
+        href = (parent.get("href") or "").strip()
+        if href and _IMG_HREF_RE.search(href):
+            cands.append(href)
+    return cands
+
+
+def _rewrite_images(root, imap):
+    """正文区外的 <img> 已由预清理删除大半；此处做白名单校验与 URL 改写。"""
+    for img in list(root.iter("img")):
+        cands = _img_candidates(img)
+        m = imap.match(cands) if cands else None
+        parent = img.getparent()
+        if m is None:
+            if parent is not None:
+                parent.remove(img)
+            continue
+        key, _oss = m
+        img.set("src", key)
+        for attr in _IMG_ATTRS:
+            if attr != "src" and img.get(attr) is not None:
+                del img.attrib[attr]
+        if img.get("srcset") is not None:
+            del img.attrib["srcset"]
+        if not (img.get("alt") or "").strip():
+            fig = parent
+            cap = None
+            if fig is not None and fig.tag == "figure":
+                for fc in fig.iter("figcaption"):
+                    cap = clean_text(fc.text_content())
+                    break
+            if cap:
+                img.set("alt", cap[:120])
+        # 裸 div 下的 img 会被 trafilatura 的通配正文路径丢弃（BODY_XPATH 不命中
+        # 哈希类名容器时 recover_wild_text 只收 p/code/quote/table 等，graphic 全灭；
+        # 实测百家号 0→19 张）。把白名单图片（或其 <a> 包装）包进 <p> 保底。
+        target = img if parent is None or parent.tag != "a" else parent
+        holder = target.getparent()
+        if holder is not None and holder.tag not in ("p", "figure", "td", "th", "li"):
+            new_p = root.makeelement("p", {})
+            target.addprevious(new_p)
+            new_p.append(target)
+
+
+def _post_md(md, imap):
+    """Markdown 后处理：图片映射 OSS、噪声行、来源标注首段、空白归一。"""
+    noise = {"strong": 0, "weak": 0, "source_label": 0}
+
+    def _img_sub(m):
+        alt, url = m.group(1), m.group(2)
+        mm = imap.match(url)
+        if mm is None:
+            return ""
+        return f"![{alt}]({mm[1]})"
+
+    md = _MD_IMG_RE.sub(_img_sub, md)
+
+    lines = md.split("\n")
+    out_lines = []
+    first_para_done = False
+    for ln in lines:
+        stripped = ln.strip()
+        if not stripped:
+            out_lines.append("")
+            continue
+        if not first_para_done:
+            first_para_done = True
+            if len(stripped) <= 40 and _SRC_LABEL_RE.match(stripped):
+                noise["source_label"] += len(stripped)
+                continue
+        if _DECLARATION_RE.match(stripped):
+            noise["declaration"] = noise.get("declaration", 0) + len(stripped)
+            continue
+        if len(stripped) <= 60 and _AI_GEN_RE.search(stripped):
+            noise["ai_gen"] = noise.get("ai_gen", 0) + len(stripped)
+            continue
+        if len(stripped) <= 50 and _APP_GUIDE_RE.search(stripped):
+            noise["app_guide"] = noise.get("app_guide", 0) + len(stripped)
+            continue
+        if len(stripped) <= 80 and _EN_CTA_RE.search(stripped):
+            noise["en_cta"] = noise.get("en_cta", 0) + len(stripped)
+            continue
+        if len(stripped) <= 30 and _LABEL_ONLY_RE.match(stripped):
+            noise["label_only"] = noise.get("label_only", 0) + len(stripped)
+            continue
+        if stripped in _STRONG_NOISE:
+            noise["strong"] += len(stripped)
+            continue
+        if mojibake_score(stripped) >= 2 and _mojibake_heavy(stripped):
+            noise["mojibake"] = noise.get("mojibake", 0) + len(stripped)
+            continue
+        if _PROMO_STRONG_RE.search(stripped):
+            noise["promo"] = noise.get("promo", 0) + len(stripped)
+            continue
+        if len(stripped) <= 60 and _PROMO_WEAK_RE.search(stripped):
+            noise["promo"] = noise.get("promo", 0) + len(stripped)
+            continue
+        if len(stripped) <= 40:
+            low = stripped.lower()
+            if any(w in low or w in stripped for w in _WEAK_NOISE):
+                noise["weak"] += len(stripped)
+                continue
+            if _FOLLOW_GUIDE_RE.match(stripped):
+                noise["weak"] += len(stripped)
+                continue
+        if _TECH_RE.match(stripped):
+            noise["weak"] += len(stripped)
+            continue
+        out_lines.append(ln.rstrip())
+
+    # 图片去重：同一 OSS 链接在正文中重复出现只保留第一次（验收反馈 case_03）
+    seen_img = set()
+    deduped = []
+    for ln in out_lines:
+        m = _MD_IMG_RE.fullmatch(ln.strip())
+        if m:
+            u = m.group(2)
+            if u in seen_img:
+                noise["dup_img"] = noise.get("dup_img", 0) + 1
+                continue
+            seen_img.add(u)
+        deduped.append(ln)
+    out_lines = deduped
+
+    # 尾部样板窗：最后 5 个非空行里的来源标注 / 纯数字计数 / UI 按钮行
+    # （"来源：京报体育 举报/反馈 0 分享" 常挂在文末，验收反馈 case_03）
+    nonempty_idx = [i for i, ln in enumerate(out_lines) if ln.strip()]
+    for i in nonempty_idx[-5:]:
+        s = out_lines[i].strip()
+        if (len(s) <= 40 and _SRC_LABEL_RE.match(s)) or \
+                (len(s) <= 6 and s.isdigit()) or s in _STRONG_NOISE:
+            noise["tail_boiler"] = noise.get("tail_boiler", 0) + len(s)
+            out_lines[i] = ""
+
+    # 相关文章区块删除：DOM 预清理对 >4000 字模块豁免，MD 层再兜一次
+    # （zhifang "## 相关文章" + 推荐列表整块漏出，验收口径：与正文无关内容须删）
+    cleaned = []
+    i = 0
+    n = len(out_lines)
+    while i < n:
+        ln = out_lines[i]
+        s = ln.strip()
+        hm = re.match(r"^(#{1,6})\s*(.+?)\s*$", s)
+        if hm and len(hm.group(2)) <= 14 and _MODULE_TITLE_RE.match(hm.group(2)):
+            j = i + 1
+            blk = []
+            while j < n and not out_lines[j].strip().startswith("#"):
+                blk.append(out_lines[j])
+                j += 1
+            nonempty = [b for b in blk if b.strip()]
+            if nonempty:
+                listy = sum(1 for b in nonempty
+                            if b.strip().startswith(("- ", "![", "* ")))
+                if listy * 2 >= len(nonempty):
+                    noise["related_block"] = noise.get("related_block", 0) + sum(
+                        len(b) for b in nonempty)
+                    i = j
+                    continue
+        cleaned.append(ln)
+        i += 1
+    out_lines = cleaned
+
+    text = "\n".join(out_lines)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip(), noise
+
+
+def _h1_tokens(h1):
+    h1 = h1.lower()
+    if has_cjk(h1):
+        s = re.sub(r"[^一-鿿]", "", h1)
+        return {s[i:i + 2] for i in range(len(s) - 1)}
+    return set(re.findall(r"[^\W\d_]{3,}", h1))
+
+
+def _h1_relevance(h1_texts, content):
+    """h1 与正文的词元重合率：低于阈值说明抽出来的是模板/推广区而非文章。"""
+    toks = set()
+    for h in h1_texts or []:
+        toks |= _h1_tokens(h)
+    if not toks:
+        return 1.0
+    c = (content or "").lower()
+    c_cjk = re.sub(r"[^一-鿿]", "", c)
+    hit = 0
+    for t in toks:
+        if has_cjk(t):
+            if t in c_cjk:
+                hit += 1
+        elif t in c:
+            hit += 1
+    return hit / len(toks)
+
+
+def _h1_region_md(root):
+    """h1 祖先链中第一个文本量 >=400 的容器，序列化其段落/列表/表格/图片。"""
+    h1 = None
+    for el in root.iter("h1"):
+        if clean_text(el.text_content()):
+            h1 = el
+            break
+    if h1 is None:
+        return ""
+    node = h1.getparent()
+    best = None
+    while node is not None:
+        if len(clean_text(node.text_content())) >= 400:
+            best = node
+            break
+        node = node.getparent()
+    if best is None:
+        return ""
+    parts = []
+    for el in best.iter():
+        if el.tag in ("p", "li"):
+            t = clean_text(el.text_content())
+            if len(t) >= 8:
+                parts.append(t)
+        elif el.tag in ("h2", "h3"):
+            t = clean_text(el.text_content())
+            if t:
+                parts.append("## " + t)
+        elif el.tag == "tr":
+            cells = [clean_text(td.text_content()) for td in el.iter("td", "th")]
+            cells = [c for c in cells if c]
+            if cells:
+                parts.append("| " + " | ".join(cells) + " |")
+        elif el.tag == "img":
+            src = el.get("src") or ""
+            if src:
+                parts.append("![%s](%s)" % (clean_text(el.get("alt") or ""), src))
+    out = []
+    for p in parts:
+        if not out or out[-1] != p:
+            out.append(p)
+    return "\n\n".join(out)
+
+
+def _title_guard_ok(text, title):
+    """标题-正文一致性守卫（只读版）：抽出文本与标题是否属同一篇。"""
+    if not text or not title:
+        return True
+    rel = _h1_relevance([title], text)
+    if rel >= 0.15:
+        return True
+    cov = _cjk_char_coverage(title, text)
+    return cov is not None and cov >= 0.5
+
+
+def _cjk_char_coverage(title, content):
+    """标题 CJK 字符在正文中的覆盖率；长标题复述型正文 bigram 低但字符覆盖率高。"""
+    t = re.sub(r"[^一-鿿]", "", title or "")
+    if not t:
+        return None
+    c = set(re.sub(r"[^一-鿿]", "", (content or "").lower()))
+    chars = set(t)
+    if not chars:
+        return None
+    return sum(1 for ch in chars if ch in c) / len(chars)
+
+
+# 非文章页 URL 模式：视频页 / 作者页 / 栏目列表页（调研05 §5：≥11% 非文章页）
+_NONARTICLE_URL_RE = re.compile(
+    r"/(video|videos|videonews|watch)(/|\.|\?|$)"
+    r"|/(author|authors|category|categories|tag|tags)/"
+    r"|[?&]a=lists\b|dtlandingsuper",
+    re.IGNORECASE,
+)
+
+
+def _drop_title_lead(content, title, pub_time=""):
+    """开头去重：首个文本行 == 标题则剔除；其后紧跟的短日期行 == pub_time 同日也剔除
+    （V3：不应把标题/发布时间作为正文首段重复输出；验收反馈 case_03/case_06）。
+    开头的图片行跳过不处理。"""
+    if not content:
+        return content
+    nt = norm_key(title) if title else ""
+    pub_day = (pub_time or "")[:10]
+    lines = content.split("\n")
+    i = 0
+    n = len(lines)
+    # 跳过开头空行与图片行
+    while i < n and (not lines[i].strip() or lines[i].strip().startswith("![")):
+        i += 1
+    dropped = False
+    if i < n and nt:
+        s2 = lines[i].strip().lstrip("#").strip()
+        if s2 and norm_key(s2) == nt:
+            del lines[i]
+            dropped = True
+            n -= 1
+    # 标题行之后（或开头）的短日期行：与 pub_time 同日即重复信息
+    j = 0
+    n = len(lines)
+    while j < n and (not lines[j].strip() or lines[j].strip().startswith("![")):
+        j += 1
+    date_dropped = False
+    if j < n and pub_day and (dropped or i == j):
+        s = lines[j].strip()
+        if len(s) <= 60:
+            tv = parse_absolute(s.lstrip("#").strip())
+            if tv and tv.dt.strftime("%Y-%m-%d") == pub_day:
+                del lines[j]
+                date_dropped = True
+    if dropped or date_dropped:
+        # 标题/日期是页面唯一文字时保留原文（zhibo8 视频页 caption==title，删了就空）
+        if any(ln.strip() and not ln.strip().startswith("![") for ln in lines):
+            return "\n".join(lines).strip()
+    return content
+
+
+def _trace_ratio(content, visible_norm):
+    if not content or not visible_norm:
+        return 0.0
+    units = []
+    for para in content.split("\n"):
+        para = para.strip().strip("|").strip()
+        if len(para) < 8:
+            continue
+        if para.startswith("!["):
+            continue
+        if "|" in para:
+            units.extend(c.strip() for c in para.split("|") if len(c.strip()) >= 8)
+        else:
+            units.append(para)
+        if len(units) >= 60:
+            break
+    if not units:
+        return 1.0
+    # 两侧必须同一套归一：unit 剥 markdown 符号（含连字符），visible 侧也要剥，
+    # 否则 'pick-up' vs 'pickup' 恒不匹配（回归实测英文正文假 0）
+    vn = re.sub(r"[*`#>\-]", "", visible_norm)
+    hit = 0
+    for u in units:
+        u = re.sub(r"[*`#>\-]", "", u)
+        if norm_key(u) and norm_key(u) in vn:
+            hit += 1
+    return hit / len(units)
+
+
+def extract_content(signals, site_cfg, page_status, images, img_ref, visible_norm,
+                    title="", url="", pub_time=""):
+    """返回 (content, content_path, content_noise_by_class, traceable_ratio)。"""
+    keep_comments = page_status in ("forum_page", "comment_page")
+    imap = _ImageMap(images, img_ref)
+
+    try:
+        root = deepcopy(signals.tree)
+    except Exception:
+        try:
+            root = lxml_html.fromstring(signals.html)
+        except Exception:
+            return "", "unparseable", {}, 0.0
+
+    try:
+        root = _preclean_tree(root, keep_comments)
+    except Exception:
+        pass
+    try:
+        from .sitespecific import site_preclean
+    except ImportError:
+        try:
+            from sitespecific import site_preclean
+        except ImportError:
+            site_preclean = None
+    if site_preclean is not None:
+        try:
+            site_preclean(getattr(signals, "host", "") or "", root)
+        except Exception:
+            pass
+    try:
+        _rewrite_images(root, imap)
+    except Exception:
+        pass
+
+    try:
+        html_str = etree.tostring(root, encoding="unicode", method="html")
+    except Exception:
+        return "", "serialize_fail", {}, 0.0
+
+    traf, cfg = _get_trafilatura()
+    md = None
+    md_rescue = None
+    path = "trafilatura_fast"
+    try:
+        md = traf.extract(
+            html_str,
+            output_format="markdown",
+            include_comments=keep_comments,
+            include_images=imap.allowed,
+            include_tables=True,
+            include_links=False,
+            favor_precision=False,
+            favor_recall=False,
+            config=cfg,
+            no_fallback=False,
+        )
+    except Exception:
+        md = None
+    if imap.allowed:
+        # trafilatura 与 readability/justext 的对比换体可能把 graphic 丢掉
+        # （百家号单图文章实测：正文 664 字时 readability 体胜出，图全灭）。
+        # 图数变少时用 recall 模式重抽一次保底，是否采用在 _post_md 之后裁决。
+        try:
+            n_tree_img = sum(1 for _ in root.iter("img"))
+            n_md_img = len(_MD_IMG_RE.findall(md or ""))
+            if n_tree_img > n_md_img:
+                md_rescue = traf.extract(
+                    html_str,
+                    output_format="markdown",
+                    include_comments=keep_comments,
+                    include_images=imap.allowed,
+                    include_tables=True,
+                    include_links=False,
+                    favor_recall=True,
+                    config=cfg,
+                    no_fallback=True,
+                )
+        except Exception:
+            md_rescue = None
+    if not md:
+        try:
+            md = traf.extract(
+                signals.html,
+                output_format="markdown",
+                include_comments=keep_comments,
+                include_images=imap.allowed,
+                include_tables=True,
+                include_links=False,
+                favor_recall=True,
+                config=cfg,
+            )
+            path = "trafilatura_recall"
+        except Exception:
+            md = None
+    if not md:
+        return "", "extract_fail", {}, 0.0
+
+    content, noise = _post_md(md, imap)
+    if md_rescue:
+        content2, noise2 = _post_md(md_rescue, imap)
+        n1 = len(_MD_IMG_RE.findall(content or ""))
+        n2 = len(_MD_IMG_RE.findall(content2 or ""))
+        l1 = len(re.sub(r"\s", "", content or ""))
+        l2 = len(re.sub(r"\s", "", content2 or ""))
+        # 换上 recall 体必须：图更多、文本量不崩、且不把标题一致性从及格打成不及格
+        # （s1#204 fstoppers：recall 体稀释相关性触发 title_mismatch 误杀）
+        if n2 > n1 and l2 >= 0.8 * max(l1, 1) and (
+                not _title_guard_ok(content, title) or _title_guard_ok(content2, title)):
+            content, noise, path = content2, noise2, path + "|img_rescue"
+    h1s = getattr(signals, "h1_texts", None) or []
+    if h1s and _h1_relevance(h1s, content) < 0.15:
+        fb = _h1_region_md(root)
+        if fb:
+            fb_content, fb_noise = _post_md(fb, imap)
+            if len(fb_content) >= 200 and _h1_relevance(h1s, fb_content) >= 0.15:
+                content, noise, path = fb_content, fb_noise, "h1_region"
+    content_pre = content
+    content = _drop_title_lead(content, title, pub_time)
+    # 标题-正文一致性守卫（V3：标题与正文必须属同一篇；错配内容不如留空）
+    # 只在文章页生效：product_page 等非文章页型的正文与标题天然弱相关（回归 s1#16-18）
+    # 判定用去标题首段之前的 content_pre：首段标题被 _drop_title_lead 移除后
+    # 重合率会假性归零（回归 s1#308）
+    if content and title and page_status in ("ok", ""):
+        # 图+短说明型页面（摄影投稿/图说）豁免：尾部样板清洗后标题重合会假性归零
+        # （回归 s1#204 fstoppers：314 字符图说被误杀；正文确为本页内容）
+        nonimg = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", content_pre)
+        img_caption = "![" in content_pre and len(re.sub(r"\s", "", nonimg)) < 400
+        if not img_caption:
+            rel = _h1_relevance([title], content_pre)
+            if rel < 0.15:
+                cov = _cjk_char_coverage(title, content_pre)
+                if cov is None or cov < 0.5:
+                    content, path = "", path + "|title_mismatch"
+            elif rel < 0.5 and _NONARTICLE_URL_RE.search(url or ""):
+                content, path = "", path + "|nonarticle_url"
+    ratio = _trace_ratio(content, visible_norm)
+    # 低溯源率守卫：抽出内容大部分无法在页面可见文本中定位，说明抽错区域
+    # （验收反馈 case_04 错位输出 ratio=0.18），错位输出不如留空
+    if content and ratio < 0.2 and not keep_comments:
+        content, path = "", path + "|low_trace"
+        ratio = _trace_ratio(content, visible_norm)
+    return content, path, noise, ratio