python-library-ff14-news 0.0.0__py3-none-any.whl

ff14_news/channels/cn_weibo/crawl_backend.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from crawl4weibo import WeiboClient
+from crawl4weibo.exceptions.base import CrawlError, NetworkError, RateLimitError
+from crawl4weibo.models.post import Post
+
+from ff14_news.channels.cn_weibo.browser_cookies import fetch_mobile_cookies
+from ff14_news.channels.cn_weibo.exceptions import WeiboAccessError
+from ff14_news.channels.cn_weibo.proxy_url import normalize_proxy_url
+
+_PLAYWRIGHT_INSTALL_HINT = (
+    "请先安装 Playwright Chromium：\n"
+    "  python -m example.ensure_browser --proxy 127.0.0.1:7897"
+)
+
+
+class WeiboCrawlBackend:
+    """基于 crawl4weibo 的微博 HTTP 后端（Playwright 取 Cookie + 移动端 API）。"""
+
+    def __init__(
+        self,
+        *,
+        cookie: str | None = None,
+        cookie_storage_path: Path | None = None,
+        browser_headless: bool = True,
+        proxy_url: str | None = None,
+    ) -> None:
+        """创建后端。
+
+        Args:
+            cookie: 浏览器 Cookie 整串；提供后不再自动开浏览器
+            cookie_storage_path: Playwright 会话缓存路径，可复用 Cookie
+            browser_headless: 自动取 Cookie 时是否无头运行 Chromium
+            proxy_url: HTTP 代理，如 ``127.0.0.1:7897`` 或 ``http://127.0.0.1:7897``
+        """
+        self._proxy = normalize_proxy_url(proxy_url)
+        if cookie:
+            cookie_value: str | dict[str, str] = cookie
+        else:
+            try:
+                cookie_value = fetch_mobile_cookies(
+                    proxy_url=self._proxy,
+                    headless=browser_headless,
+                    storage_state_path=cookie_storage_path,
+                )
+            except Exception as exc:
+                raise _access_error(exc) from exc
+            if not cookie_value:
+                raise WeiboAccessError(
+                    "Playwright 未获取到微博 Cookie，请检查代理或手动配置 Cookie。"
+                )
+        try:
+            self._client = WeiboClient(
+                cookies=cookie_value,
+                auto_fetch_cookies=False,
+                use_browser_cookies=False,
+                log_level="WARNING",
+            )
+        except Exception as exc:
+            raise _access_error(exc) from exc
+        if self._proxy:
+            self._client.add_proxy(self._proxy)
+
+    def resolve_screen_name(self, screen_name: str) -> str:
+        """按 screen_name 解析 numeric uid。"""
+        name = screen_name.strip()
+        if not name:
+            raise ValueError("screen_name must not be empty")
+        try:
+            users = self._client.search_users(name, count=20)
+        except (CrawlError, NetworkError, RateLimitError) as exc:
+            raise _access_error(exc) from exc
+        lowered = name.lower()
+        for user in users:
+            if user.screen_name.lower() == lowered:
+                return str(user.id)
+        raise ValueError(f"cannot resolve uid for screen_name={name!r}")
+
+    def fetch_timeline_posts(self, uid: str, *, page: int) -> list[Post]:
+        """拉取用户时间线一页。"""
+        if page < 1:
+            raise ValueError("page must be >= 1")
+        try:
+            return self._client.get_user_posts(
+                uid,
+                page=page,
+                expand=False,
+            )
+        except (CrawlError, NetworkError, RateLimitError) as exc:
+            raise _access_error(exc) from exc
+
+    def fetch_post(self, article_id: str) -> Post:
+        """拉取单条微博（列表级字段，不展开长文）。"""
+        article_id = str(article_id).strip()
+        if not article_id:
+            raise ValueError("article_id must not be empty")
+        try:
+            return self._client.get_post_by_bid(article_id, expand=False)
+        except (CrawlError, NetworkError, RateLimitError) as exc:
+            raise _access_error(exc) from exc
+
+    def fetch_post_detail(self, article_id: str) -> Post:
+        """拉取单条微博详情（展开长文）。"""
+        article_id = str(article_id).strip()
+        if not article_id:
+            raise ValueError("article_id must not be empty")
+        try:
+            return self._client.get_post_by_bid(article_id, expand=True)
+        except (CrawlError, NetworkError, RateLimitError) as exc:
+            raise _access_error(exc) from exc
+
+
+def _access_error(exc: Exception) -> WeiboAccessError:
+    message = str(exc).strip()
+    lowered = message.lower()
+    if "playwright" in lowered or "executable" in lowered or "chromium" in lowered:
+        return WeiboAccessError(
+            f"微博需要 Playwright Chromium。{_PLAYWRIGHT_INSTALL_HINT}"
+        )
+    if "432" in message:
+        return WeiboAccessError(
+            "微博返回 HTTP 432。"
+            f"请确认代理可用（如 127.0.0.1:7897），或配置 example/weibo_cookie.txt。"
+        )
+    if isinstance(exc, RateLimitError):
+        return WeiboAccessError(f"微博限流：{message}")
+    return WeiboAccessError(message or "微博请求失败")

ff14_news/channels/cn_weibo/exceptions.py

@@ -0,0 +1,2 @@
+class WeiboAccessError(ValueError):
+    """微博接口拒绝访问或会话初始化失败。"""

ff14_news/channels/cn_weibo/mblog_parser.py

@@ -0,0 +1,161 @@
+import html
+import re
+from datetime import datetime
+
+from ff14_news.channels.cn_weibo.constants import (
+    DETAIL_URL_TEMPLATE,
+    MOBILE_ORIGIN,
+    PERMALINK_TEMPLATE,
+)
+from ff14_news.common.html_blocks import html_to_blocks
+from ff14_news.models import NewsArticle, NewsBlockType, NewsContentBlock, NewsListItem
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_FULL_TEXT_RE = re.compile(r"…?\.\.\.?全文\s*$|…全文\s*$")
+_TITLE_MAX_LEN = 80
+_SUMMARY_MAX_LEN = 200
+
+
+def effective_mblog(mblog: dict) -> dict:
+    """转发微博取内层原博；普通微博返回自身。"""
+    inner = mblog.get("retweeted_status")
+    if isinstance(inner, dict):
+        return inner
+    return mblog
+
+
+def needs_detail_fetch(mblog: dict) -> bool:
+    """正文被截断或标记长文时须拉 statuses/show。"""
+    if mblog.get("isLongText"):
+        return True
+    raw = str(mblog.get("text") or "")
+    plain = _plain_text(raw)
+    if "全文" in raw and ("status" in raw or _FULL_TEXT_RE.search(plain)):
+        return True
+    return False
+
+
+def mblog_id(mblog: dict) -> str:
+    value = mblog.get("id") or mblog.get("mid") or mblog.get("bid")
+    if not value:
+        raise ValueError("mblog missing id")
+    return str(value)
+
+
+def parse_created_at(raw: str) -> datetime:
+    text = (raw or "").strip()
+    if not text:
+        return datetime.fromtimestamp(0)
+    try:
+        return datetime.strptime(text, "%a %b %d %H:%M:%S %z %Y")
+    except ValueError:
+        return datetime.fromtimestamp(0)
+
+
+def mblog_to_list_item(mblog: dict, *, channel_id: str) -> NewsListItem:
+    article_id = mblog_id(mblog)
+    effective = effective_mblog(mblog)
+    title = _title_from_text(effective)
+    summary = _summary_from_text(effective)
+    cover = _first_pic_url(effective)
+    return NewsListItem(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=parse_created_at(str(mblog.get("created_at") or "")),
+        summary=summary,
+        cover_image_url=cover,
+        source_page_url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+    )
+
+
+def mblog_to_article(
+    mblog: dict,
+    *,
+    channel_id: str,
+) -> NewsArticle:
+    article_id = mblog_id(mblog)
+    effective = effective_mblog(mblog)
+    blocks = blocks_from_mblog(mblog)
+    title = _title_from_text(effective)
+    summary = _summary_from_blocks(blocks) or _summary_from_text(effective)
+    cover = _first_pic_url(effective)
+    return NewsArticle(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=parse_created_at(str(mblog.get("created_at") or "")),
+        summary=summary,
+        category_code=None,
+        cover_image_url=cover,
+        source_page_url=PERMALINK_TEMPLATE.format(article_id=article_id),
+        blocks=blocks,
+    )
+
+
+def blocks_from_mblog(mblog: dict) -> list[NewsContentBlock]:
+    effective = effective_mblog(mblog)
+    text_html = str(effective.get("text") or "")
+    wrapped = f"<div>{text_html}</div>" if text_html else ""
+    blocks = html_to_blocks(wrapped, base_url=MOBILE_ORIGIN)
+    for pic in effective.get("pics") or []:
+        url = _pic_url(pic)
+        if url:
+            blocks.append(
+                NewsContentBlock(type=NewsBlockType.IMAGE, url=url)
+            )
+    return blocks
+
+
+def _plain_text(text_html: str) -> str:
+    text = _TAG_RE.sub("", text_html)
+    return html.unescape(text).replace("\xa0", " ").strip()
+
+
+def _title_from_text(mblog: dict) -> str:
+    plain = _plain_text(str(mblog.get("text") or ""))
+    first_line = plain.split("\n", 1)[0].strip()
+    if not first_line:
+        return "微博"
+    if len(first_line) <= _TITLE_MAX_LEN:
+        return first_line
+    return first_line[: _TITLE_MAX_LEN - 1] + "…"
+
+
+def _summary_from_text(mblog: dict) -> str:
+    plain = _plain_text(str(mblog.get("text") or ""))
+    plain = _FULL_TEXT_RE.sub("", plain).strip()
+    if len(plain) <= _SUMMARY_MAX_LEN:
+        return plain
+    return plain[: _SUMMARY_MAX_LEN - 1] + "…"
+
+
+def _summary_from_blocks(blocks: list[NewsContentBlock]) -> str:
+    for block in blocks:
+        if block.text and block.text.strip():
+            text = block.text.strip()
+            if len(text) <= _SUMMARY_MAX_LEN:
+                return text
+            return text[: _SUMMARY_MAX_LEN - 1] + "…"
+    return ""
+
+
+def _first_pic_url(mblog: dict) -> str | None:
+    pics = mblog.get("pics") or []
+    if not pics:
+        return None
+    return _pic_url(pics[0])
+
+
+def _pic_url(pic: object) -> str | None:
+    if not isinstance(pic, dict):
+        return None
+    large = pic.get("large")
+    if isinstance(large, dict):
+        url = large.get("url")
+        if url:
+            return str(url).strip() or None
+    url = pic.get("url")
+    if url:
+        return str(url).strip() or None
+    return None

ff14_news/channels/cn_weibo/post_adapter.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from crawl4weibo.models.post import Post
+
+from ff14_news.channels.cn_weibo.constants import (
+    DETAIL_URL_TEMPLATE,
+    PERMALINK_TEMPLATE,
+)
+from ff14_news.channels.cn_weibo.mblog_parser import (
+    _SUMMARY_MAX_LEN,
+    _TITLE_MAX_LEN,
+    _FULL_TEXT_RE,
+)
+from ff14_news.models import NewsArticle, NewsBlockType, NewsContentBlock, NewsListItem
+
+
+def post_to_list_item(post: Post, *, channel_id: str) -> NewsListItem:
+    """将 crawl4weibo Post 转为列表级 NewsListItem。"""
+    effective = _effective_post(post)
+    article_id = str(post.id).strip()
+    title = _title_from_plain(effective.text)
+    summary = _summary_from_plain(effective.text)
+    cover = effective.pic_urls[0] if effective.pic_urls else None
+    return NewsListItem(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=_publish_date(post.created_at),
+        summary=summary,
+        cover_image_url=cover,
+        source_page_url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+    )
+
+
+def post_to_article(post: Post, *, channel_id: str) -> NewsArticle:
+    """将 crawl4weibo Post 转为含正文块的 NewsArticle。"""
+    effective = _effective_post(post)
+    article_id = str(post.id).strip()
+    blocks = _blocks_from_post(effective)
+    title = _title_from_plain(effective.text)
+    summary = _summary_from_blocks(blocks) or _summary_from_plain(effective.text)
+    cover = effective.pic_urls[0] if effective.pic_urls else None
+    return NewsArticle(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=_publish_date(post.created_at),
+        summary=summary,
+        category_code=None,
+        cover_image_url=cover,
+        source_page_url=PERMALINK_TEMPLATE.format(article_id=article_id),
+        blocks=blocks,
+    )
+
+
+def _effective_post(post: Post) -> Post:
+    if post.retweeted_status is not None:
+        return post.retweeted_status
+    return post
+
+
+def _publish_date(value: datetime | None) -> datetime:
+    if value is None:
+        return datetime.fromtimestamp(0, tz=timezone.utc)
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value
+
+
+def _title_from_plain(text: str) -> str:
+    first_line = text.strip().split("\n", 1)[0].strip()
+    if not first_line:
+        return "微博"
+    if len(first_line) <= _TITLE_MAX_LEN:
+        return first_line
+    return first_line[: _TITLE_MAX_LEN - 1] + "…"
+
+
+def _summary_from_plain(text: str) -> str:
+    plain = _FULL_TEXT_RE.sub("", text.strip()).strip()
+    if len(plain) <= _SUMMARY_MAX_LEN:
+        return plain
+    return plain[: _SUMMARY_MAX_LEN - 1] + "…"
+
+
+def _summary_from_blocks(blocks: list[NewsContentBlock]) -> str:
+    for block in blocks:
+        if block.text and block.text.strip():
+            snippet = block.text.strip()
+            if len(snippet) <= _SUMMARY_MAX_LEN:
+                return snippet
+            return snippet[: _SUMMARY_MAX_LEN - 1] + "…"
+    return ""
+
+
+def _blocks_from_post(post: Post) -> list[NewsContentBlock]:
+    blocks: list[NewsContentBlock] = []
+    text = post.text.strip()
+    if text:
+        blocks.append(NewsContentBlock(type=NewsBlockType.TEXT, text=text))
+    for url in post.pic_urls:
+        blocks.append(NewsContentBlock(type=NewsBlockType.IMAGE, url=url))
+    return blocks

ff14_news/channels/cn_weibo/profile.py

@@ -0,0 +1,10 @@
+def weibo_timeline_container_id(uid: str) -> str:
+    """用户微博时间线 containerid（固定规则）。
+
+    Args:
+        uid: 微博 numeric uid
+    """
+    uid = str(uid).strip()
+    if not uid:
+        raise ValueError("uid must not be empty")
+    return f"107603{uid}"

ff14_news/channels/cn_weibo/proxy_url.py

@@ -0,0 +1,14 @@
+def normalize_proxy_url(value: str | None) -> str | None:
+    """将 ``host:port`` 或完整 URL 规范为 ``http://…`` 代理地址。
+
+    Args:
+        value: 代理地址；空串或 None 表示不使用代理
+    """
+    if value is None:
+        return None
+    text = value.strip()
+    if not text:
+        return None
+    if "://" not in text:
+        return f"http://{text}"
+    return text

ff14_news/channels/jp_official/__init__.py

@@ -0,0 +1,3 @@
+from ff14_news.channels.jp_official.channel import JpOfficialChannel
+
+__all__ = ["JpOfficialChannel"]

ff14_news/channels/jp_official/channel.py

@@ -0,0 +1,110 @@
+from datetime import datetime, timezone
+
+from ff14_news.channels.jp_official.constants import (
+    CHANNEL_ID,
+    DETAIL_URL_TEMPLATE,
+    DISPLAY_NAME,
+    TOPICS_LIST_URL,
+)
+from ff14_news.channels.jp_official.detail_parser import (
+    parse_detail_metadata,
+    parse_detail_page,
+)
+from ff14_news.channels.jp_official.http_client import fetch_html
+from ff14_news.channels.jp_official.list_parser import (
+    list_row_to_item,
+    parse_topics_list_page,
+    topics_list_url,
+)
+from ff14_news.common.list_feed import article_from_list_item
+from ff14_news.models import NewsArticle, NewsFeed, NewsListItem
+
+_LIST_SCAN_PAGE_SIZE = 30
+_LIST_SCAN_MAX_PAGES = 20
+
+
+class JpOfficialChannel:
+    """FF14 日文官网 Lodestone トピックス。
+
+    默认抓取列表级字段（含 news__list--banner 摘要）；正文块须 fetch_article_detail。
+    """
+
+    channel_id = CHANNEL_ID
+    display_name = DISPLAY_NAME
+
+    def __init__(self, *, timeout_seconds: float = 120.0) -> None:
+        self._timeout = timeout_seconds
+
+    def list_items(
+        self,
+        *,
+        limit: int = 10,
+        page_index: int = 0,
+    ) -> list[NewsListItem]:
+        if limit < 1:
+            raise ValueError("limit must be >= 1")
+        url = topics_list_url(page_index)
+        html = fetch_html(url, timeout_seconds=self._timeout)
+        rows = parse_topics_list_page(html, limit=limit)
+        return [list_row_to_item(row, channel_id=self.channel_id) for row in rows]
+
+    def fetch_article_detail(self, article_id: str) -> NewsArticle:
+        article_id = str(article_id).strip()
+        if not article_id:
+            raise ValueError("article_id must not be empty")
+        url = DETAIL_URL_TEMPLATE.format(article_id=article_id)
+        html = fetch_html(url, timeout_seconds=self._timeout)
+        return parse_detail_page(html, article_id, channel_id=self.channel_id)
+
+    def fetch_article(self, article_id: str) -> NewsArticle:
+        return self.fetch_article_detail(article_id)
+
+    def fetch_articles(
+        self,
+        *,
+        limit: int = 10,
+        page_index: int = 0,
+    ) -> NewsFeed:
+        items = self.list_items(limit=limit, page_index=page_index)
+        articles = [article_from_list_item(item) for item in items]
+        return self._build_feed(articles)
+
+    def fetch_articles_by_ids(self, article_ids: list[str]) -> NewsFeed:
+        if not article_ids:
+            raise ValueError("article_ids must not be empty")
+        wanted = {str(aid).strip() for aid in article_ids}
+        found: dict[str, NewsArticle] = {}
+        for page_index in range(_LIST_SCAN_MAX_PAGES):
+            if wanted.issubset(found):
+                break
+            items = self.list_items(
+                limit=_LIST_SCAN_PAGE_SIZE,
+                page_index=page_index,
+            )
+            if not items:
+                break
+            for item in items:
+                if item.id in wanted and item.id not in found:
+                    found[item.id] = article_from_list_item(item)
+        articles: list[NewsArticle] = []
+        for aid in article_ids:
+            key = str(aid).strip()
+            if key in found:
+                articles.append(found[key])
+            else:
+                articles.append(self._metadata_from_detail(key))
+        return self._build_feed(articles)
+
+    def _metadata_from_detail(self, article_id: str) -> NewsArticle:
+        url = DETAIL_URL_TEMPLATE.format(article_id=article_id)
+        html = fetch_html(url, timeout_seconds=self._timeout)
+        return parse_detail_metadata(html, article_id, channel_id=self.channel_id)
+
+    def _build_feed(self, articles: list[NewsArticle]) -> NewsFeed:
+        return NewsFeed(
+            channel_id=self.channel_id,
+            source_list_url=TOPICS_LIST_URL,
+            category_code=None,
+            fetched_at=datetime.now(timezone.utc),
+            articles=articles,
+        )

ff14_news/channels/jp_official/constants.py

@@ -0,0 +1,6 @@
+CHANNEL_ID = "jp_official"
+DISPLAY_NAME = "FF14 日文官网（Lodestone）"
+
+SITE_ORIGIN = "https://jp.finalfantasyxiv.com"
+TOPICS_LIST_URL = f"{SITE_ORIGIN}/lodestone/topics/"
+DETAIL_URL_TEMPLATE = f"{SITE_ORIGIN}/lodestone/topics/detail/{{article_id}}/"

ff14_news/channels/jp_official/detail_parser.py

@@ -0,0 +1,128 @@
+import html
+import re
+from datetime import datetime, timezone
+
+from ff14_news.channels.jp_official.constants import DETAIL_URL_TEMPLATE, SITE_ORIGIN
+from ff14_news.common.html_blocks import html_to_blocks
+from ff14_news.models import NewsArticle, NewsContentBlock
+
+_WRAPPER_RE = re.compile(
+    r'<div class="news__detail__wrapper">(.*?)</div>\s*<div class="news__detail__social">',
+    re.DOTALL,
+)
+_TITLE_RE = re.compile(
+    r'<article class="news__detail">.*?<h1>([^<]+)</h1>',
+    re.DOTALL,
+)
+_TIMESTAMP_RE = re.compile(
+    r'<article class="news__detail">.*?ldst_strftime\((\d+),',
+    re.DOTALL,
+)
+_FIRST_IMG_RE = re.compile(r'<img[^>]+src="([^"]+)"')
+
+
+def parse_detail_metadata(
+    page_html: str,
+    article_id: str,
+    *,
+    channel_id: str,
+) -> NewsArticle:
+    """详情页元数据：标题、时间、头图、摘要，不解析正文块。"""
+    title_match = _TITLE_RE.search(page_html)
+    title = html.unescape(title_match.group(1).strip()) if title_match else ""
+
+    ts_match = _TIMESTAMP_RE.search(page_html)
+    if ts_match:
+        publish_date = datetime.fromtimestamp(
+            int(ts_match.group(1)),
+            tz=timezone.utc,
+        )
+    else:
+        publish_date = datetime.fromtimestamp(0, tz=timezone.utc)
+
+    wrapper_match = _WRAPPER_RE.search(page_html)
+    wrapper_html = wrapper_match.group(1) if wrapper_match else ""
+    cover_url = _first_image_url(wrapper_html)
+    summary = _plain_summary_from_html(wrapper_html)
+
+    return NewsArticle(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=publish_date,
+        summary=summary,
+        category_code=None,
+        cover_image_url=cover_url,
+        source_page_url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+        blocks=[],
+    )
+
+
+def parse_detail_page(
+    page_html: str,
+    article_id: str,
+    *,
+    channel_id: str,
+) -> NewsArticle:
+    title_match = _TITLE_RE.search(page_html)
+    title = html.unescape(title_match.group(1).strip()) if title_match else ""
+
+    ts_match = _TIMESTAMP_RE.search(page_html)
+    if ts_match:
+        publish_date = datetime.fromtimestamp(
+            int(ts_match.group(1)),
+            tz=timezone.utc,
+        )
+    else:
+        publish_date = datetime.fromtimestamp(0, tz=timezone.utc)
+
+    wrapper_match = _WRAPPER_RE.search(page_html)
+    wrapper_html = wrapper_match.group(1) if wrapper_match else ""
+    blocks = html_to_blocks(wrapper_html, base_url=SITE_ORIGIN)
+
+    cover_url = _first_image_url(wrapper_html)
+    summary = _summary_from_blocks(blocks)
+
+    return NewsArticle(
+        channel_id=channel_id,
+        id=article_id,
+        title=title,
+        publish_date=publish_date,
+        summary=summary,
+        category_code=None,
+        cover_image_url=cover_url,
+        source_page_url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+        blocks=blocks,
+    )
+
+
+def _first_image_url(wrapper_html: str) -> str | None:
+    match = _FIRST_IMG_RE.search(wrapper_html)
+    if not match:
+        return None
+    url = match.group(1).strip()
+    return url or None
+
+
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _plain_summary_from_html(wrapper_html: str, max_len: int = 200) -> str:
+    text = _TAG_RE.sub(" ", wrapper_html)
+    text = html.unescape(text)
+    text = re.sub(r"\s+", " ", text).strip()
+    if not text:
+        return ""
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 1] + "…"
+
+
+def _summary_from_blocks(blocks: list[NewsContentBlock], max_len: int = 200) -> str:
+    for block in blocks:
+        if block.text and block.text.strip():
+            text = block.text.strip()
+            if len(text) <= max_len:
+                return text
+            return text[: max_len - 1] + "…"
+    return ""

ff14_news/channels/jp_official/http_client.py

@@ -0,0 +1,16 @@
+import urllib.error
+import urllib.request
+
+_USER_AGENT = "Mozilla/5.0 (compatible; python-library-ff14-news/0.1)"
+
+
+def fetch_html(url: str, *, timeout_seconds: float) -> str:
+    req = urllib.request.Request(
+        url,
+        headers={"User-Agent": _USER_AGENT, "Accept": "text/html"},
+    )
+    try:
+        raw = urllib.request.urlopen(req, timeout=timeout_seconds).read()
+    except urllib.error.HTTPError as exc:
+        raise ValueError(f"HTTP {exc.code} for {url}") from exc
+    return raw.decode("utf-8", "replace")