rolling-reader 0.3.0__tar.gz → 0.4.0__tar.gz

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: rolling-reader
-Version: 0.3.0
+Version: 0.4.0
 Summary: Local-first web scraper that automatically rolls through HTTP → browser → JS state extraction
 License: MIT
 Requires-Python: >=3.11
 Requires-Dist: beautifulsoup4>=4.14
 Requires-Dist: httpx>=0.28
 Requires-Dist: playwright>=1.44
+Requires-Dist: trafilatura>=1.12
 Requires-Dist: typer>=0.12
 Description-Content-Type: text/markdown
 

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "rolling-reader"
-version = "0.3.0"
+version = "0.4.0"
 description = "Local-first web scraper that automatically rolls through HTTP → browser → JS state extraction"
 readme = "README.md"
 license = { text = "MIT" }
@@ -14,6 +14,7 @@ dependencies = [
     "beautifulsoup4>=4.14",
     "typer>=0.12",
     "playwright>=1.44",
+    "trafilatura>=1.12",
 ]
 
 [project.scripts]

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/src/rolling_reader/cli.py

@@ -90,6 +90,11 @@ def scrape(
         "--verbose", "-v",
         help="Print escalation steps to stderr",
     ),
+    clean: bool = typer.Option(
+        False,
+        "--clean", "-c",
+        help="Extract article body only, filtering out navigation, ads, and footers",
+    ),
 ) -> None:
     """Scrape a URL and output structured data."""
 
@@ -101,6 +106,7 @@ def scrape(
                 cdp_endpoint=cdp_endpoint,
                 verbose=verbose,
                 use_cache=not no_cache,
+                clean=clean,
             )
         )
     except ExtractionError as e:

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/src/rolling_reader/dispatcher.py

@@ -38,6 +38,7 @@ async def dispatch(
     page_timeout: float = 30.0,
     verbose: bool = False,
     use_cache: bool = True,
+    clean: bool = False,
 ) -> ExtractResult:
     """
     自动选择最优抓取策略并执行。
@@ -64,11 +65,11 @@ async def dispatch(
     # ── 强制指定层级 ──────────────────────────────────────────────────────
     if force_level == 1:
         log("forced Level 1 (HTTP)")
-        return await http_extract(url, timeout=http_timeout)
+        return await http_extract(url, timeout=http_timeout, clean=clean)
 
     if force_level in (2, 3):
         log(f"forced Level 2/3 (CDP)")
-        return await _try_level2(url, cdp_endpoint, page_timeout, log)
+        return await _try_level2(url, cdp_endpoint, page_timeout, log, clean=clean)
 
     # ── Profile Cache：命中时直接跳到已知层级 ─────────────────────────────
     if use_cache:
@@ -78,7 +79,7 @@ async def dispatch(
             log(f"cache hit → Level {preferred} for {cached.get('domain')}")
             if preferred == 1:
                 try:
-                    result = await http_extract(url, timeout=http_timeout)
+                    result = await http_extract(url, timeout=http_timeout, clean=clean)
                     profile_cache.save(url, result.level)
                     return result
                 except Exception:
@@ -86,7 +87,7 @@ async def dispatch(
                     profile_cache.invalidate(url)
             else:
                 try:
-                    result = await _try_level2(url, cdp_endpoint, page_timeout, log)
+                    result = await _try_level2(url, cdp_endpoint, page_timeout, log, clean=clean)
                     profile_cache.save(url, result.level,
                                        state_var=cached.get("state_var"))
                     return result
@@ -99,7 +100,7 @@ async def dispatch(
     # Level 1：HTTP 直取
     log(f"Level 1 → {url}")
     try:
-        result = await http_extract(url, timeout=http_timeout)
+        result = await http_extract(url, timeout=http_timeout, clean=clean)
         log(f"Level 1 succeeded ({result.elapsed_ms:.0f}ms)")
         if use_cache:
             profile_cache.save(url, result.level)
@@ -112,7 +113,7 @@ async def dispatch(
         log(f"Level 1 → error ({e.reason}), escalating to Level 2/3")
 
     # Level 2/3：CDP + 已有 Chrome（内部自动尝试 Level 3 state 提取）
-    result = await _try_level2(url, cdp_endpoint, page_timeout, log)
+    result = await _try_level2(url, cdp_endpoint, page_timeout, log, clean=clean)
     if use_cache:
         state_var = None
         if result.level == 3:
@@ -127,6 +128,8 @@ async def _try_level2(
     cdp_endpoint: str,
     page_timeout: float,
     log,
+    *,
+    clean: bool = False,
 ) -> ExtractResult:
     """尝试 Level 2，Chrome 不可用时给出清晰错误。"""
     from rolling_reader.extractor.cdp import ChromeNotRunningError
@@ -146,6 +149,7 @@ async def _try_level2(
             url,
             cdp_endpoint=cdp_endpoint,
             page_timeout=page_timeout,
+            clean=clean,
         )
         log(f"Level 2 succeeded ({result.elapsed_ms:.0f}ms)")
         return result

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/src/rolling_reader/extractor/cdp.py

@@ -73,6 +73,7 @@ async def extract(
     cdp_endpoint: str = CDP_ENDPOINT,
     page_timeout: float = 30.0,
     wait_networkidle: bool = True,
+    clean: bool = False,
 ) -> ExtractResult:
     """
     Level 2 CDP 抓取。
@@ -175,12 +176,18 @@ async def extract(
 
     # ── 9. Level 2：回退到 DOM 提取 ───────────────────────────────────────
     soup = BeautifulSoup(html, "html.parser")
+    if clean:
+        from rolling_reader.extractor.clean import clean_extract
+        cleaned = clean_extract(html, url=final_url)
+        text = cleaned if cleaned else _extract_text(BeautifulSoup(html, "html.parser"))
+    else:
+        text = _extract_text(BeautifulSoup(html, "html.parser"))
     return ExtractResult(
         url=final_url,
         level=2,
         status_code=200,
         title=_extract_title(soup),
-        text=_extract_text(BeautifulSoup(html, "html.parser")),
+        text=text,
         links=_extract_links(soup, final_url),
         elapsed_ms=round(elapsed, 1),
     )

rolling_reader-0.4.0/src/rolling_reader/extractor/clean.py

@@ -0,0 +1,44 @@
+"""
+rolling_reader/extractor/clean.py
+==================================
+正文提取（Article Extraction）
+
+使用 trafilatura 从 HTML 中识别并提取主体文章内容，
+过滤导航栏、广告、页脚、侧边栏等噪音。
+
+对比默认的 BeautifulSoup 文本提取：
+  默认：把 <body> 里所有文字全部返回（快，但夹杂噪音）
+  --clean：只返回主体文章文字（慢约 50ms，但干净）
+"""
+
+from __future__ import annotations
+from typing import Optional
+
+
+def clean_extract(html: str, url: str = "") -> Optional[str]:
+    """
+    从 HTML 中提取正文。
+
+    Args:
+        html: 完整 HTML 字符串
+        url:  原始 URL（trafilatura 用于辅助判断，可选）
+
+    Returns:
+        正文文字，或 None（trafilatura 无法识别正文时）
+    """
+    try:
+        import trafilatura
+    except ImportError:
+        raise ImportError(
+            "trafilatura is required for --clean mode: pip install trafilatura"
+        )
+
+    text = trafilatura.extract(
+        html,
+        url=url or None,
+        include_comments=False,
+        include_tables=True,
+        no_fallback=False,   # 允许回退到其他算法
+        favor_precision=True,
+    )
+    return text or None

{rolling_reader-0.3.0 → rolling_reader-0.4.0}/src/rolling_reader/extractor/http.py

@@ -182,6 +182,7 @@ async def extract(
     timeout: float = 15.0,
     headers: Optional[dict] = None,
     client: Optional[httpx.AsyncClient] = None,
+    clean: bool = False,
 ) -> ExtractResult:
     """
     Level 1 HTTP 抓取。
@@ -219,9 +220,16 @@ async def extract(
         # 解析内容
         soup = BeautifulSoup(response.text, "html.parser")
         title = _extract_title(soup)
-        text  = _extract_text(BeautifulSoup(response.text, "html.parser"))  # 用新 soup 避免修改影响
         links = _extract_links(soup, str(response.url))
 
+        # --clean 模式：用 trafilatura 替换 BeautifulSoup 文本提取
+        if clean:
+            from rolling_reader.extractor.clean import clean_extract
+            cleaned = clean_extract(response.text, url=str(response.url))
+            text = cleaned if cleaned else _extract_text(soup)
+        else:
+            text = _extract_text(BeautifulSoup(response.text, "html.parser"))
+
         return ExtractResult(
             url=str(response.url),
             level=1,