newscli-tool 1.0.0__py3-none-any.whl

newscli/__init__.py

@@ -0,0 +1,6 @@
+"""
+newscli/__init__.py — news-cli package entry
+"""
+
+__version__ = "1.0.0"
+__all__ = ["__version__"]

newscli/aggregator.py

@@ -0,0 +1,263 @@
+"""
+aggregator.py — News Aggregator
+
+统一调度多个 source，支持：
+- 多 source 并行拉取（ThreadPoolExecutor）
+- 每个 source 的 module 子模块指定
+- source-specific 额外参数（category, language, node, subreddit 等）
+- 统一 JSON 输出（完整 schema）
+"""
+
+import concurrent.futures
+from dataclasses import asdict
+from typing import Optional
+from newscli.sources import REGISTRY, NewsItem, SourceError, rss as rss_module
+
+
+class NewsAggregator:
+    """多 source 统一聚合器"""
+
+    def __init__(
+        self,
+        sources: list[str] | None = None,
+        limit_per_source: int = 10,
+    ):
+        """
+        Args:
+            sources         : source 名称列表（如 ["hackernews", "github"]）
+                              None = 所有注册 source（不含 rss）
+            limit_per_source: 每个 source 最大拉取条数
+        """
+        self.sources = sources or list(REGISTRY.keys())
+        self.limit_per_source = limit_per_source
+
+    def fetch(
+        self,
+        source_filter: str | None = None,
+        limit: int | None = None,
+        keyword: str | None = None,
+        params: dict | None = None,
+        enrich: bool = True,  # 默认对 summary=null 的 item 拉取原文摘要
+    ) -> dict:
+        """
+        并行拉取所有 source，返回统一格式。
+
+        Args:
+            source_filter : 逗号分隔的 source:module 对列表
+                           如 "hackernews:topstories,github:trending,v2ex:latest"
+            limit         : 全局返回上限（None = 所有）
+            keyword       : 关键词过滤（所有 source 生效）
+            params        : source-specific 参数 dict
+                           格式：{"<source>": {"module": "...", "category": "..."}}
+
+        Returns:
+            {"ok": bool, "schema": str, "sources": {}, "items": [], "total": int, "errors": []}
+            items 每项 = NewsItem.to_dict()，即所有字段（含 None）
+        """
+        parsed = self._parse_filter(source_filter)
+        params = params or {}
+
+        # results key = source:module（如 "hackernews:topstories"）
+        results: dict[str, list[dict]] = {}
+        errors: list[str] = []
+
+        def _fetch_one(name: str, module: str | None, extra: dict):
+            """执行单次 fetch，返回 (key, items_list, errors_list)"""
+            # 唯一 key：不同 module 的同一 source 不会互相覆盖
+            key = f"{name}:{module}" if module else name
+
+            # RSS 特殊处理（不注册到 REGISTRY）
+            if name == "rss":
+                src = rss_module.RSSSource(
+                    source_key=extra.get("source_key"),
+                    feed_url=extra.get("url"),
+                )
+            else:
+                src_cls = REGISTRY.get(name)
+                if not src_cls:
+                    return key, [], [f"Unknown source: {name}"]
+                src = src_cls()
+
+            # global keyword filter applied at source.fetch() call
+            fetch_kwargs = {
+                "module": module,
+                "limit": self.limit_per_source,
+                "keyword": keyword,   # may be None = no filtering
+            }
+            # per-source params override (from merged extra)
+            for k in ("category", "language", "node", "subreddit",
+                      "start_time", "end_time", "url"):
+                if k in extra:
+                    fetch_kwargs[k] = extra[k]
+            # per-source keyword (params level) takes precedence over global
+            if "keyword" in extra:
+                fetch_kwargs["keyword"] = extra["keyword"]
+
+            try:
+                items = src.fetch(**fetch_kwargs)
+                return key, [item.to_dict() for item in items], []
+            except SourceError as e:
+                return key, [], [str(e)]
+            except Exception as e:
+                return key, [], [f"{name} unexpected error: {e}"]
+
+        # 构建 work items
+        work: list[tuple] = []
+        if parsed:
+            for name, module, extra in parsed:
+                # CLI params 覆盖 hard-coded extra
+                sp = params.get(name, {})
+                merged_extra = {**extra, **sp}
+                work.append((name, module, merged_extra))
+        else:
+            for name in self.sources:
+                sp = params.get(name, {})
+                work.append((name, None, sp))
+
+        # 并行执行
+        with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(work), 8)) as ex:
+            futures = {ex.submit(_fetch_one, *w): w for w in work}
+            for fut in concurrent.futures.as_completed(futures):
+                key, items, errs = fut.result()
+                if items:
+                    results[key] = items
+                errors.extend(errs)
+
+        # 合并所有 items（按 source:module 分别存储，输出时全部扁平化）
+        all_items: list[dict] = []
+        for key, items in results.items():
+            all_items.extend(items)
+
+        # 跨 source 去重：标题相似度 ≥ 70%（仅多 source 时触发）
+        if len(results) > 1:
+            all_items = self._deduplicate(all_items)
+
+        # 可选：enrich — 对 summary=null 的 item 并发拉取原文 description
+        if enrich:
+            from .enrich import enrich_items
+            all_items = enrich_items(all_items)
+
+        if limit is not None:
+            all_items = all_items[:limit]
+
+        return {
+            "ok": True,
+            "schema": "NewsItem v1.0",
+            "sources": {k: len(v) for k, v in results.items()},
+            "items": all_items,
+            "total": len(all_items),
+            "errors": errors,
+        }
+
+    @staticmethod
+    def _deduplicate(items: list[dict]) -> list[dict]:
+        """
+        跨 source 去重。相似度阈值 70%（标题 normalized 后比对）。
+        保留第一条出现的item，移除后续相似项。
+        """
+        from urllib.parse import urlparse
+        def normalize_title(t: str) -> str:
+            """小写 + 去除标点 + strip()"""
+            import re
+            t = t.lower().strip()
+            t = re.sub(r'[^\w\s]', ' ', t)
+            t = re.sub(r'\s+', ' ', t).strip()
+            return t
+
+        def similarity(a: str, b: str) -> float:
+            """简单词集合 Jaccard 相似度"""
+            sa = set(a.split())
+            sb = set(b.split())
+            if not sa or not sb:
+                return 0.0
+            inter = len(sa & sb)
+            union = len(sa | sb)
+            return inter / union if union > 0 else 0.0
+
+        def item_key(item: dict) -> str:
+            domain = ""
+            if item.get("url"):
+                try:
+                    domain = urlparse(item["url"]).netloc
+                except Exception:
+                    pass
+            return f"{normalize_title(item['title'])}|{domain}"
+
+        seen: list[dict] = []
+        for item in items:
+            norm = normalize_title(item.get("title", ""))
+            if not norm:
+                seen.append(item)
+                continue
+            dup_idx = None
+            for i, s in enumerate(seen):
+                s_norm = normalize_title(s.get("title", ""))
+                # Different domain → different article, skip
+                s_domain = urlparse(s.get("url", "")).netloc or ""
+                item_domain = urlparse(item.get("url", "")).netloc or ""
+                if s_domain and item_domain and s_domain != item_domain:
+                    continue
+                if similarity(norm, s_norm) >= 0.70:
+                    dup_idx = i
+                    break
+            if dup_idx is not None:
+                # 保留信息更丰富的那个（非 null 字段数量多的）
+                existing = seen[dup_idx]
+                existing_nulls = sum(1 for k, v in existing.items() if v is None and k != 'extra')
+                item_nulls = sum(1 for k, v in item.items() if v is None and k != 'extra')
+                if item_nulls < existing_nulls:
+                    seen[dup_idx] = item
+            else:
+                seen.append(item)
+        return seen
+
+    @staticmethod
+    def _parse_filter(filter_str: str | None) -> list[tuple]:
+        """
+        解析 'source:module,source2:module2' → [(name, module, extra_dict), ...]
+
+        支持格式：
+          hackernews:topstories        → ("hackernews", "topstories", {})
+          v2ex:node:python             → ("v2ex", "node", {"node": "python"})
+          rss:bensbites                → ("rss", "bensbites", {"source_key": "bensbites"})
+          github:trending:language=Python → ("github", "trending", {"language": "Python"})
+          zaker:category:category=technology → ("zaker", "category", {"category": "technology"})
+          reddit:r/technology          → ("reddit", None, {"subreddit": "technology"})
+        """
+        if not filter_str:
+            return []
+        items = []
+        for part in filter_str.split("&"):
+            part = part.strip()
+            if not part:
+                continue
+            name, rest = part.split(":", 1) if ":" in part else (part, "")
+            name = name.strip()
+
+            extra = {}
+            tokens = rest.split(":") if rest else []
+            module = None
+            for token in tokens:
+                if "=" in token:
+                    k, v = token.split("=", 1)
+                    extra[k.strip()] = v.strip()
+                elif not module:
+                    module = token
+
+            # Reddit r/<subreddit> shorthand
+            if name == "reddit" and module and module.startswith("r/"):
+                extra["subreddit"] = module[2:]
+                module = None
+
+            # RSS source_key shorthand
+            if name == "rss" and module and "=" not in module:
+                extra["source_key"] = module
+
+            items.append((name, module, extra))
+        return items
+
+
+def fetch_all(keyword: str | None = None, limit: int | None = None) -> dict:
+    """快速入口：拉取所有注册 source"""
+    agg = NewsAggregator()
+    return agg.fetch(keyword=keyword, limit=limit)

newscli/cli.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""
+cli.py — News CLI 入口
+
+两种接口共存：
+  1. flag 模式（--source / --limit / --json）— 机器友好
+  2. 自然语言模式（get hackernews topstories 5 json）— 人类友好
+
+用法（flag）：
+    python cli.py --source hackernews:topstories --limit 5
+    python cli.py --source all --json
+
+用法（自然语言）：
+    python cli.py get hackernews topstories
+    python cli.py get hackernews topstories 5 json
+    python cli.py get hackernews topstories 5 and github trending 10
+    python cli.py get all 10 json noenrich
+    python cli.py list
+    python cli.py list hackernews
+"""
+
+import argparse
+import json
+import sys
+from .aggregator import NewsAggregator
+from .sources import REGISTRY, rss as rss_module
+from .parser import parseNL, ParseError, list_sources, list_source_modules
+
+
+# ─── 输出格式器 ─────────────────────────────────────────────────
+
+def _render_text(items: list[dict]) -> str:
+    """人类可读文本输出 — None 字段不显示"""
+    if not items:
+        return "📰 无内容"
+    lines = [f"📰 共 {len(items)} 条"]
+    lines.append("─" * 50)
+    for i, item in enumerate(items, 1):
+        # source + module badge
+        module = item.get("module") or ""
+        source = item.get("source") or "?"
+        badge = f"[{source}]" if not module else f"[{source}/{module}]"
+        heat = item.get("heat") or ""
+        time = item.get("time") or ""
+        meta = " | ".join(x for x in [heat, time] if x)
+        lines.append(f"\n[{i}] {badge} {meta}")
+        lines.append(f"    {item.get('title', '')}")
+        url = item.get("url")
+        if url:
+            lines.append(f"    🔗 {url}")
+        summary = item.get("summary")
+        if summary:
+            lines.append(f"    📝 {summary[:150]}{'...' if len(summary) > 150 else ''}")
+        author = item.get("author")
+        if author:
+            lines.append(f"    👤 {author}")
+    return "\n".join(lines)
+
+
+def _json_output(result: dict) -> str:
+    """JSON 输出 — 完整 schema，包含 None"""
+    return json.dumps(result, indent=2, ensure_ascii=False)
+
+
+# ─── CLI 参数解析 ───────────────────────────────────────────────
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="news",
+        description="模块化新闻聚合 CLI — 支持 HN/GitHub/HuggingFace/ZAKER/V2EX/Reddit/RSS",
+    )
+    p.add_argument(
+        "--source", "-s", default="all",
+        help="source:module 格式，逗号分隔多源\n"
+             "  hackernews:topstories|newest|ask|show|jobs\n"
+             "  github:trending\n"
+             "  huggingface:daily|trending\n"
+             "  zaker:hot|category|search\n"
+             "  v2ex:hot|latest|node:<name>\n"
+             "  reddit:popular|hot|r/<subreddit>\n"
+             "  rss:<preset>|custom\n"
+             "  all = 所有注册 source（不含 rss）"
+    )
+    p.add_argument("--limit", "-n", type=int, default=10, help="每 source 最大条数")
+    p.add_argument(
+        "--keyword", "-k", default=None,
+        help="关键词过滤（逗号分隔多词，AND 匹配）"
+    )
+    p.add_argument(
+        "--params", "-p", default=None,
+        help="source-specific 参数，JSON 格式\n"
+             '  如: {"zaker": {"category": "technology"}, "github": {"language": "Python"}}'
+    )
+    p.add_argument("--json", action="store_true", help="JSON 输出（供 agent 解析）")
+    p.add_argument(
+        "--enrich", "-e", action="store_true", default=True,
+        help="对 summary=null 的 item 并发拉取原文 og:description（curl，8s 超时，默认开启）"
+    )
+    p.add_argument(
+        "--no-enrich", dest="enrich", action="store_false",
+        help="禁用 --enrich，不拉取原文摘要"
+    )
+    p.add_argument(
+        "--modules", action="store_true",
+        help="列出所有 source 的可用 module"
+    )
+    return p
+
+
+def _parse_params(params_str: str | None) -> dict:
+    """解析 --params JSON 字符串"""
+    if not params_str:
+        return {}
+    try:
+        return json.loads(params_str)
+    except Exception:
+        return {}
+
+
+# ─── 主入口 ─────────────────────────────────────────────────────
+
+def main() -> None:
+    # ── 自然语言模式检测 ──
+    # 如果没有任何 flag，且 positional args 以 list/get/fetch/看/拉/找 开头，走 NL parser
+    if len(sys.argv) > 1 and not sys.argv[1].startswith("-"):
+        raw = " ".join(sys.argv[1:])
+        try:
+            result = _run_nl(raw)
+            sys.exit(0 if result["ok"] else 1)
+        except ParseError as e:
+            print(f"❗ 语法错误：{e}")
+            print(f"提示：get hackernews topstories 5  /  list hackernews")
+            sys.exit(1)
+        except Exception as e:
+            print(f"❗ 执行出错：{e}")
+            sys.exit(1)
+
+    # ── Flag 模式（原有行为）──
+    parser = _build_parser()
+    args = parser.parse_args()
+
+    if args.modules:
+        _print_modules()
+        return
+
+    source_filter = None
+    if args.source != "all":
+        source_filter = args.source
+
+    params = _parse_params(args.params)
+    agg = NewsAggregator(limit_per_source=args.limit)
+    result = agg.fetch(
+        source_filter=source_filter,
+        limit=None,
+        keyword=args.keyword,
+        params=params,
+        enrich=args.enrich,
+    )
+
+    if args.json:
+        print(_json_output(result))
+    else:
+        print(_render_text(result["items"]))
+        if result["errors"]:
+            print(f"\n⚠️ {len(result['errors'])} 个 source 出错：")
+            for e in result["errors"]:
+                print(f"  - {e}")
+
+
+def _run_nl(raw: str) -> dict:
+    """运行自然语言命令。"""
+    result = parseNL(raw)
+
+    if result.command == "list":
+        if result.list_target == "sources":
+            print(list_sources())
+        elif result.list_target == "modules":
+            # 列出所有源的所有模块
+            for src in REGISTRY.keys():
+                print(list_source_modules(src))
+                print()
+        else:
+            print(list_source_modules(result.list_target))
+        return {"ok": True, "items": []}
+
+    # ── fetch 模式 ──
+    f = result.fetch
+    source_filter = f["source_filter"] if f["source_filter"] != "all" else None
+
+    agg = NewsAggregator(limit_per_source=f["limit"])
+    agg_result = agg.fetch(
+        source_filter=source_filter,
+        limit=None,
+        keyword=f["keyword"],
+        params=f["params"],
+        enrich=f["enrich"],
+    )
+
+    if f["output"] == "json":
+        print(_json_output(agg_result))
+    else:
+        print(_render_text(agg_result["items"]))
+        if agg_result["errors"]:
+            print(f"\n⚠️ {len(agg_result['errors'])} 个 source 出错：")
+            for e in agg_result["errors"]:
+                print(f"  - {e}")
+
+    return agg_result
+
+
+def _print_modules() -> None:
+    """打印所有 source 及其支持的 module"""
+    all_sources = dict(REGISTRY)
+    all_sources["rss"] = rss_module.RSSSource
+
+    print(f"{'Source':<15} {'Display Name':<25} {'Modules'}")
+    print("─" * 80)
+
+    # RSS presets
+    for key, info in sorted(rss_module.PRESET_SOURCES.items()):
+        print(f"{'rss:'+key:<15} {info['name']:<25} (RSS preset)")
+
+    print()
+    for name, cls in sorted(all_sources.items()):
+        if name == "rss":
+            continue
+        modules = ", ".join(cls.modules) if cls.modules else "(all)"
+        print(f"{name:<15} {cls.display_name:<25} {modules}")
+
+
+if __name__ == "__main__":
+    main()

newscli/enrich.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+enrich.py — URL enrichment: fetch og:description / meta description via curl
+
+对 summary=null 的 item 并发拉取原文 description，填充 summary 字段。
+使用 curl（subprocess）绕过 Python requests 对某些站点的 TLS 超时问题。
+"""
+
+import concurrent.futures
+import re
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass
+class EnrichResult:
+    url: str
+    description: str | None  # None = failed/not found
+    error: str | None
+
+
+def fetch_description(url: str, timeout: int = 8) -> EnrichResult:
+    """
+    用 curl 获取页面 meta og:description，失败返回 None。
+    超时视为失败，不阻塞。
+    """
+    try:
+        result = subprocess.run(
+            [
+                "curl", "-s", "-L", "--max-time", str(timeout),
+                "-A", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "-H", "Accept: text/html,application/xhtml+xml,*/*",
+                "-H", "Accept-Language: en-US,en;q=0.9",
+                url,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=timeout + 1,  # process-level timeout slightly higher
+        )
+        html = result.stdout
+        # og:description first (richer)
+        m = _extract_description(html)
+        return EnrichResult(url=url, description=m, error=None)
+    except subprocess.TimeoutExpired:
+        return EnrichResult(url=url, description=None, error="timeout")
+    except Exception as e:
+        return EnrichResult(url=url, description=None, error=str(e))
+
+
+def _extract_description(html: str) -> str | None:
+    """从 HTML 中提取 og:description 或 meta description"""
+    patterns = [
+        # og:description
+        r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\'](.*?)["\']',
+        r'<meta[^>]+content=["\'](.*?)["\'][^>]+property=["\']og:description["\']',
+        # meta description
+        r'<meta[^>]+name=["\']description["\'][^>]+content=["\'](.*?)["\']',
+        r'<meta[^>]+content=["\'](.*?)["\'][^>]+name=["\']description["\']',
+    ]
+    for pattern in patterns:
+        m = re.search(pattern, html, re.IGNORECASE)
+        if m:
+            text = m.group(1).strip()
+            # Clean HTML entities
+            text = text.replace("&amp;", "&").replace("&quot;", '"').replace("&#x27;", "'").replace("&lt;", "<").replace("&gt;", ">").replace("&#x2F;", "/").replace("&nbsp;", " ")
+            if text:
+                return text[:500]
+    return None
+
+
+def enrich_items(items: list[dict], max_workers: int = 8) -> list[dict]:
+    """
+    对所有 summary=null 的 item 并发拉取 description。
+
+    Args:
+        items: NewsItem.to_dict() 列表
+        max_workers: 并发线程数
+
+    Returns:
+        同输入结构，summary 已填充的 item（不改变顺序）
+    """
+    # Build work: index → (item, url)
+    null_items = [(i, items[i]) for i in range(len(items)) if items[i].get("summary") is None]
+
+    if not null_items:
+        return items
+
+    urls_to_fetch = [(i, item["url"]) for i, item in null_items if item.get("url")]
+
+    if not urls_to_fetch:
+        return items
+
+    results_map: dict[int, str | None] = {}  # index → description
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=min(max_workers, len(urls_to_fetch))) as ex:
+        futures = {ex.submit(fetch_description, url): idx for idx, url in urls_to_fetch}
+        for fut in concurrent.futures.as_completed(futures):
+            idx = futures[fut]
+            try:
+                res = fut.result()
+                results_map[idx] = res.description
+            except Exception:
+                results_map[idx] = None
+
+    # Merge back
+    enriched = []
+    for i, item in enumerate(items):
+        if i in results_map and results_map[i]:
+            item = {**item, "summary": results_map[i]}
+        enriched.append(item)
+
+    return enriched