python-library-ff14-news 0.0.0__py3-none-any.whl

ff14_news/channels/jp_official/list_parser.py

@@ -0,0 +1,112 @@
+import html
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from ff14_news.channels.jp_official.constants import DETAIL_URL_TEMPLATE
+from ff14_news.models import NewsListItem
+
+_SUMMARY_MAX_LEN = 200
+_TAG_RE = re.compile(r"<[^>]+>")
+_DETAIL_ID_RE = re.compile(r"/lodestone/topics/detail/([a-f0-9]+)/?")
+_BANNER_RE = re.compile(
+    r'<div class="news__list--banner">(.*?)</div>\s*(?=</li>|<header)',
+    re.DOTALL,
+)
+_TITLE_RE = re.compile(
+    r'class="news__list--title"[^>]*>\s*<a[^>]*>([^<]+)</a>',
+    re.DOTALL,
+)
+_TIMESTAMP_RE = re.compile(r"ldst_strftime\((\d+),")
+_COVER_RE = re.compile(
+    r'class="news__list--img"[^>]*>\s*<img[^>]+src="([^"]+)"',
+    re.DOTALL,
+)
+_ITEM_SPLIT_RE = re.compile(r'<li class="news__list--topics[^"]*">')
+
+
+@dataclass(frozen=True)
+class TopicsListRow:
+    article_id: str
+    title: str
+    publish_date: datetime
+    summary: str
+    cover_image_url: str | None
+
+
+def topics_list_url(page_index: int) -> str:
+    from ff14_news.channels.jp_official.constants import TOPICS_LIST_URL
+
+    if page_index <= 0:
+        return TOPICS_LIST_URL
+    return f"{TOPICS_LIST_URL}?page={page_index + 1}"
+
+
+def parse_topics_list_page(html: str, *, limit: int) -> list[TopicsListRow]:
+    fragments = _ITEM_SPLIT_RE.split(html)
+    rows: list[TopicsListRow] = []
+    for fragment in fragments[1:]:
+        row = _parse_item_fragment(fragment)
+        if row is not None:
+            rows.append(row)
+        if len(rows) >= limit:
+            break
+    return rows
+
+
+def list_row_to_item(row: TopicsListRow, *, channel_id: str) -> NewsListItem:
+    return NewsListItem(
+        channel_id=channel_id,
+        id=row.article_id,
+        title=row.title,
+        publish_date=row.publish_date,
+        summary=row.summary,
+        cover_image_url=row.cover_image_url,
+        source_page_url=DETAIL_URL_TEMPLATE.format(article_id=row.article_id),
+    )
+
+
+def _parse_item_fragment(fragment: str) -> TopicsListRow | None:
+    id_match = _DETAIL_ID_RE.search(fragment)
+    if not id_match:
+        return None
+    article_id = id_match.group(1)
+
+    title_match = _TITLE_RE.search(fragment)
+    title = html.unescape(title_match.group(1).strip()) if title_match else ""
+
+    ts_match = _TIMESTAMP_RE.search(fragment)
+    if ts_match:
+        publish_date = datetime.fromtimestamp(
+            int(ts_match.group(1)),
+            tz=timezone.utc,
+        )
+    else:
+        publish_date = datetime.fromtimestamp(0, tz=timezone.utc)
+
+    cover_match = _COVER_RE.search(fragment)
+    cover = cover_match.group(1).strip() if cover_match else None
+    summary = _banner_plain_summary(fragment)
+
+    return TopicsListRow(
+        article_id=article_id,
+        title=title,
+        publish_date=publish_date,
+        summary=summary,
+        cover_image_url=cover,
+    )
+
+
+def _banner_plain_summary(fragment: str) -> str:
+    match = _BANNER_RE.search(fragment)
+    if not match:
+        return ""
+    inner = re.sub(r"<img[^>]*>", " ", match.group(1), flags=re.IGNORECASE)
+    text = _TAG_RE.sub(" ", inner)
+    text = html.unescape(text)
+    text = re.sub(r"\s+", " ", text).strip()
+    if not text:
+        return ""
+    if len(text) <= _SUMMARY_MAX_LEN:
+        return text
+    return text[: _SUMMARY_MAX_LEN - 1] + "…"

ff14_news/common/html_blocks.py

@@ -0,0 +1,183 @@
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+from ff14_news.models import NewsBlockType, NewsContentBlock
+
+_BLOCK_TAGS = frozenset(
+    {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr", "img", "br"}
+)
+_SKIP_TAGS = frozenset({"style", "script", "head", "meta", "link", "noscript"})
+_HEADING_LEVEL = {"h1": 1, "h2": 2, "h3": 3, "h4": 4, "h5": 5, "h6": 6}
+
+
+class _ContentHTMLParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.blocks: list[NewsContentBlock] = []
+        self._skip_depth = 0
+        self._text_buf: list[str] = []
+        self._stack: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip_depth += 1
+            return
+        if self._skip_depth:
+            return
+        self._stack.append(tag)
+        if tag == "img":
+            src = _attr(attrs, "src")
+            if src:
+                self._flush_text()
+                self.blocks.append(
+                    NewsContentBlock(
+                        type=NewsBlockType.IMAGE,
+                        url=src,
+                        text=_attr(attrs, "alt"),
+                    )
+                )
+            return
+        if tag == "br":
+            self._text_buf.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip_depth = max(0, self._skip_depth - 1)
+            return
+        if self._skip_depth:
+            return
+        if self._stack and self._stack[-1] == tag:
+            self._stack.pop()
+        if tag in _HEADING_LEVEL:
+            self._emit_text_block(NewsBlockType.HEADING, _HEADING_LEVEL[tag])
+            return
+        if tag == "p" or tag == "li":
+            self._emit_text_block(NewsBlockType.TEXT, None)
+            return
+        if tag == "tr":
+            self._emit_text_block(NewsBlockType.TEXT, None, join_cells=True)
+            return
+
+    def handle_data(self, data: str) -> None:
+        if self._skip_depth:
+            return
+        if self._stack and self._stack[-1] == "img":
+            return
+        stripped = data.replace("\xa0", " ")
+        if stripped.strip():
+            self._text_buf.append(stripped)
+
+    def close(self) -> None:
+        super().close()
+        if not self._skip_depth:
+            self._flush_text()
+
+    def _emit_text_block(
+        self,
+        block_type: NewsBlockType,
+        level: int | None,
+        *,
+        join_cells: bool = False,
+    ) -> None:
+        text = "".join(self._text_buf).strip()
+        self._text_buf.clear()
+        if not text:
+            return
+        if join_cells:
+            text = " | ".join(part.strip() for part in text.split("\n") if part.strip())
+        self.blocks.append(
+            NewsContentBlock(type=block_type, text=text, level=level)
+        )
+
+    def _flush_text(self) -> None:
+        self._emit_text_block(NewsBlockType.TEXT, None)
+
+
+def html_to_blocks(
+    html: str,
+    *,
+    base_url: str,
+    extra_boilerplate: frozenset[str] | None = None,
+) -> list[NewsContentBlock]:
+    """将 HTML 片段转为有序正文块。"""
+    parser = _ContentHTMLParser()
+    parser.feed(html or "")
+    parser.close()
+    return _normalize_blocks(
+        parser.blocks,
+        base_url=base_url,
+        extra_boilerplate=extra_boilerplate,
+    )
+
+
+def _normalize_blocks(
+    blocks: list[NewsContentBlock],
+    *,
+    base_url: str,
+    extra_boilerplate: frozenset[str] | None,
+) -> list[NewsContentBlock]:
+    out: list[NewsContentBlock] = []
+    for block in blocks:
+        if block.type == NewsBlockType.IMAGE and block.url:
+            url = block.url.strip()
+            if not url.startswith(("http://", "https://")):
+                url = urljoin(base_url, url)
+            alt = (block.text or "").strip() or None
+            out.append(
+                NewsContentBlock(type=NewsBlockType.IMAGE, url=url, text=alt)
+            )
+            continue
+        text = (block.text or "").strip()
+        if not text:
+            continue
+        if block.type == NewsBlockType.TEXT and _is_boilerplate(
+            text, extra_boilerplate
+        ):
+            continue
+        out.append(
+            NewsContentBlock(
+                type=block.type,
+                text=text,
+                level=block.level,
+                url=block.url,
+            )
+        )
+    return _merge_adjacent_text(out)
+
+
+def _merge_adjacent_text(blocks: list[NewsContentBlock]) -> list[NewsContentBlock]:
+    merged: list[NewsContentBlock] = []
+    for block in blocks:
+        if (
+            merged
+            and block.type == NewsBlockType.TEXT
+            and merged[-1].type == NewsBlockType.TEXT
+        ):
+            prev = merged[-1]
+            merged[-1] = NewsContentBlock(
+                type=NewsBlockType.TEXT,
+                text=f"{prev.text}\n{block.text}",
+            )
+            continue
+        merged.append(block)
+    return merged
+
+
+def _is_boilerplate(text: str, extra: frozenset[str] | None) -> bool:
+    lowered = text.lower()
+    if lowered in {"分享到：", "分享到:"}:
+        return True
+    if "copyright" in lowered and "square enix" in lowered:
+        return True
+    if extra and text.strip() in extra:
+        return True
+    return False
+
+
+def _attr(attrs: list[tuple[str, str | None]], name: str) -> str | None:
+    for key, value in attrs:
+        if key.lower() == name and value:
+            return value.strip()
+    return None

ff14_news/common/list_feed.py

@@ -0,0 +1,20 @@
+from ff14_news.models import NewsArticle, NewsListItem
+
+
+def article_from_list_item(
+    item: NewsListItem,
+    *,
+    category_code: int | None = None,
+) -> NewsArticle:
+    """列表项转文章：保留列表级字段，正文块为空。"""
+    return NewsArticle(
+        channel_id=item.channel_id,
+        id=item.id,
+        title=item.title,
+        publish_date=item.publish_date,
+        summary=item.summary,
+        category_code=category_code,
+        cover_image_url=item.cover_image_url,
+        source_page_url=item.source_page_url,
+        blocks=[],
+    )

ff14_news/ff14_news.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+
+from ff14_news.channel_protocol import NewsChannel
+from ff14_news.channels.cn_official import CnOfficialChannel
+from ff14_news.channels.cn_weibo import CnWeiboChannel
+from ff14_news.channels.jp_official import JpOfficialChannel
+
+
+class FF14News:
+    """FF14 新闻聚合门面：各渠道独立实现，通过属性访问。"""
+
+    def __init__(
+        self,
+        *,
+        cn_official_timeout_seconds: float = 60.0,
+        cn_weibo_timeout_seconds: float = 60.0,
+        cn_weibo_cookie: str | None = None,
+        cn_weibo_cookie_storage_path: Path | str | None = None,
+        cn_weibo_browser_headless: bool = True,
+        cn_weibo_proxy_url: str | None = None,
+        jp_official_timeout_seconds: float = 120.0,
+    ) -> None:
+        """聚合各渠道实例。
+
+        Args:
+            cn_official_timeout_seconds: 国服官网 HTTP 超时
+            cn_weibo_timeout_seconds: 微博渠道 HTTP 超时
+            cn_weibo_cookie: 微博 m.weibo.cn Cookie 整串；未提供时用 Playwright 自动获取
+            cn_weibo_cookie_storage_path: Playwright 会话缓存路径
+            cn_weibo_browser_headless: 微博自动取 Cookie 时是否无头浏览器
+            cn_weibo_proxy_url: 微博 HTTP 代理，如 ``127.0.0.1:7897``
+            jp_official_timeout_seconds: 日文 Lodestone HTTP 超时
+        """
+        self.cn_official = CnOfficialChannel(
+            timeout_seconds=cn_official_timeout_seconds,
+        )
+        weibo_storage = (
+            Path(cn_weibo_cookie_storage_path).expanduser()
+            if cn_weibo_cookie_storage_path is not None
+            else None
+        )
+        self.cn_weibo = CnWeiboChannel(
+            timeout_seconds=cn_weibo_timeout_seconds,
+            cookie=cn_weibo_cookie,
+            cookie_storage_path=weibo_storage,
+            browser_headless=cn_weibo_browser_headless,
+            proxy_url=cn_weibo_proxy_url,
+        )
+        self.jp_official = JpOfficialChannel(
+            timeout_seconds=jp_official_timeout_seconds,
+        )
+
+    def available_channels(self) -> list[str]:
+        return ["cn_official", "cn_weibo", "jp_official"]
+
+    def channel(self, channel_id: str) -> NewsChannel:
+        if channel_id == "cn_official":
+            return self.cn_official
+        if channel_id == "cn_weibo":
+            return self.cn_weibo
+        if channel_id == "jp_official":
+            return self.jp_official
+        known = ", ".join(self.available_channels())
+        raise KeyError(f"unknown channel {channel_id!r}; known: {known}")

ff14_news/models.py

@@ -0,0 +1,74 @@
+from datetime import datetime
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class NewsBlockType(StrEnum):
+    """正文块类型。"""
+
+    TEXT = "text"
+    HEADING = "heading"
+    IMAGE = "image"
+
+
+class NewsContentBlock(BaseModel):
+    """单条有序正文块：纯文本、标题或图片。"""
+
+    type: NewsBlockType = Field(description="块类型")
+    text: str | None = Field(default=None, description="文本或标题内容")
+    url: str | None = Field(default=None, description="图片绝对地址")
+    level: int | None = Field(default=None, description="标题级别，1–6")
+
+
+class NewsArticle(BaseModel):
+    """一篇新闻。
+
+    默认 fetch_articles / fetch_articles_by_ids 仅填充列表级字段，blocks 为空。
+    正文块须通过各渠道 fetch_article_detail 拉取。
+    """
+
+    channel_id: str = Field(description="渠道标识，如 cn_official")
+    id: str = Field(description="渠道内文章 ID")
+    title: str = Field(description="标题")
+    publish_date: datetime = Field(description="发布时间")
+    summary: str = Field(default="", description="摘要")
+    category_code: int | None = Field(
+        default=None,
+        description="栏目编号；仅部分渠道有（如国服 cqnews）",
+    )
+    cover_image_url: str | None = Field(default=None, description="列表头图")
+    source_page_url: str = Field(description="官网详情页 hash 链接")
+    blocks: list[NewsContentBlock] = Field(
+        default_factory=list,
+        description="按阅读顺序排列的正文块",
+    )
+
+
+class NewsFeed(BaseModel):
+    """一次抓取结果：列表顺序与对应渠道新闻列表一致。"""
+
+    channel_id: str = Field(description="渠道标识，如 cn_official")
+    source_list_url: str = Field(description="列表页地址")
+    category_code: int | None = Field(
+        default=None,
+        description="列表栏目编号；仅部分渠道有",
+    )
+    fetched_at: datetime = Field(description="抓取完成时间")
+    articles: list[NewsArticle] = Field(
+        default_factory=list,
+        description="文章列表，顺序与列表 API 返回一致",
+    )
+
+
+class NewsListItem(BaseModel):
+    """列表项元数据（未展开正文）。"""
+
+    channel_id: str = Field(description="渠道标识，如 cn_official")
+    id: str = Field(description="文章 ID")
+    title: str = Field(description="标题")
+    publish_date: datetime = Field(description="发布时间")
+    summary: str = Field(default="", description="摘要")
+    cover_image_url: str | None = Field(default=None, description="头图")
+    source_page_url: str = Field(description="官网详情页链接")

python_library_ff14_news-0.0.0.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: python-library-ff14-news
+Version: 0.0.0
+Requires-Python: >=3.10
+Requires-Dist: crawl4weibo>=0.1.0
+Requires-Dist: pydantic>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'

python_library_ff14_news-0.0.0.dist-info/RECORD

@@ -0,0 +1,31 @@
+ff14_news/__init__.py,sha256=IZTOJge_RhqdAGpNwUbbWBKH_v7D5qTHcwJ4cVsR3sg,639
+ff14_news/channel_protocol.py,sha256=p5dYCQK93jnRUQGrg6VvDCkR1FJkNbKFCXhoEVejA3Q,1260
+ff14_news/ff14_news.py,sha256=eLz_8F7HCOiDCiJvbGqV-fmP3esqjIcvuX5j8G90ACU,2627
+ff14_news/models.py,sha256=d8FrQcoBxWv5UEE7PdKr7Zd9-82w5PqrYfXpFjb46j8,2785
+ff14_news/channels/__init__.py,sha256=SLvlzIcCrPiGxHh1DzMEZMa4myiTb_coAk7GUHVEBXM,71
+ff14_news/channels/cn_official/__init__.py,sha256=JT6_rlJaSE5BUf7jmCGIvBfzx34DyYVi3S_Ua-6D1Oc,105
+ff14_news/channels/cn_official/channel.py,sha256=YxPuHJczyZ8gFMKBGc7q_GJ-M-N0Gw2Op3z5xDlIn4c,3986
+ff14_news/channels/cn_official/constants.py,sha256=9XfWkEFes82wOmKnuWraZeIlvgzB_BTLziRTIyz2pc4,429
+ff14_news/channels/cn_official/cqnews_client.py,sha256=TpTLOVHgM_OP9LewvY9BQGryxQ6sZGh5-aqyev-yBLs,3829
+ff14_news/channels/cn_official/html_content.py,sha256=kb8bfMwDXbnOnOlQLnnlQMKBjvNNrhs2m2k7kazniLI,359
+ff14_news/channels/cn_weibo/__init__.py,sha256=8jhON62o1XsjUI8_3IN9p58a4YXXnLeE1z448ZFwxDQ,96
+ff14_news/channels/cn_weibo/browser_cookies.py,sha256=IOi592r6mmNjeipgOMMhtMm9A8CcTeCEyRxOWCXZmik,3161
+ff14_news/channels/cn_weibo/channel.py,sha256=YyGkvBK5ahh8FiegHw8UnqY7t45Zy1gaqwwm6AW_r44,5184
+ff14_news/channels/cn_weibo/constants.py,sha256=ss0Zt27u_XPM1N7Dtei4MV6okB8LcM1VxIT53jwDBqA,363
+ff14_news/channels/cn_weibo/crawl_backend.py,sha256=y_vd_gkywgyOCG2fkabFqgY-BxkNMp0op8kgpfCGmhg,5165
+ff14_news/channels/cn_weibo/exceptions.py,sha256=A4BiOwzzR56yjmUE99xv0MOImoeyHMyjrHm4_HrI8ME,100
+ff14_news/channels/cn_weibo/mblog_parser.py,sha256=ORohcZtToeDJJaXl9b13BxaIPBADT8XzKFfg8fbS53g,4965
+ff14_news/channels/cn_weibo/post_adapter.py,sha256=bE7MdJYkEs61vusmGnpt2p5kkc2iLtd5FFvrlqg5aUc,3509
+ff14_news/channels/cn_weibo/profile.py,sha256=8bntCHpR94Lu2Y54npss2NKD7_K3LXQ5_aMG464myio,292
+ff14_news/channels/cn_weibo/proxy_url.py,sha256=0a7p-yICIJ6ICeCV_H9cVEx3058S2pOljId1IOuWGt8,416
+ff14_news/channels/jp_official/__init__.py,sha256=7ro_82cfJgYJbGnyLWqRlcX21-o2wz_4Gwf8f418Nzw,105
+ff14_news/channels/jp_official/channel.py,sha256=M0f3QSMfdYE_S7J8KMymlR8nsCl4mqx4ggJ1D7VhGW0,4021
+ff14_news/channels/jp_official/constants.py,sha256=xh1PL70D69k7V1j2QZrMsBEilp7Fa7THxig3Jeft8uQ,263
+ff14_news/channels/jp_official/detail_parser.py,sha256=jr8VEjjzf_degkiYKe04aH6Mz5g2NTphbv_ZS78DojM,3897
+ff14_news/channels/jp_official/http_client.py,sha256=u4JqzRilOwouTLnc4amFqYF8RYn25p0dBZjbZHNyqbI,544
+ff14_news/channels/jp_official/list_parser.py,sha256=bJ8sA3GQe_JFEVAp3igmpx7yJBGHmGbR0XVG8pvcPy4,3392
+ff14_news/common/html_blocks.py,sha256=W3kz0hlti30Jc3R8lOIk43wevK5QDrO4KRHx84519jw,5778
+ff14_news/common/list_feed.py,sha256=9-p3RRTD9g2zukeLvSJP7dlidrMl1hUHS-lUe23hYrI,596
+python_library_ff14_news-0.0.0.dist-info/METADATA,sha256=UCq3aY8-HaIgyIRUqHceKidUkjZOEGF93rNvxlniZ-U,218
+python_library_ff14_news-0.0.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+python_library_ff14_news-0.0.0.dist-info/RECORD,,

python_library_ff14_news-0.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any