nonebot-plugin-bililive 2.0.4__tar.gz → 2.0.6__tar.gz

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nonebot-plugin-bililive
-Version: 2.0.4
+Version: 2.0.6
 Summary: Push bilibili dynamics and live notifications to QQ with NoneBot2.
 Keywords: nonebot,nonebot2,nonebot-plugin,qqbot,bilibili
 Author-Email: SK-415 <2967923486@qq.com>

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/database/db.py

@@ -21,6 +21,10 @@ class DB:
 
     _ready = False
 
+    @classmethod
+    def get_dynamic_offset_path(cls) -> Path:
+        return Path(get_path("dynamic_offset.json"))
+
     @classmethod
     async def init(cls):
         """初始化数据库"""
@@ -46,11 +50,14 @@ class DB:
         await Tortoise.generate_schemas()
         await cls.migrate()
         await cls.update_uid_list()
+        await cls.load_dynamic_offsets()
+        await cls.save_dynamic_offsets()
         cls._ready = True
 
     @classmethod
     async def close(cls):
         cls._ready = False
+        await cls.save_dynamic_offsets()
         await connections.close_all()
 
     @classmethod
@@ -68,6 +75,43 @@ class DB:
 
         return cls._ready
 
+    @classmethod
+    async def load_dynamic_offsets(cls):
+        path = cls.get_dynamic_offset_path()
+        if not path.exists():
+            return
+        try:
+            raw_offsets = json.loads(path.read_text(encoding="utf-8"))
+        except Exception:
+            logger.warning("动态偏移量缓存读取失败，将使用当前内存状态继续运行")
+            return
+
+        for uid_str, value in raw_offsets.items():
+            try:
+                uid = int(uid_str)
+                dynamic_id = int(value)
+            except (TypeError, ValueError):
+                continue
+            if uid in dynamic_offset:
+                dynamic_offset[uid] = dynamic_id
+
+    @classmethod
+    async def save_dynamic_offsets(cls):
+        path = cls.get_dynamic_offset_path()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        serialized_offsets = {
+            str(uid): dynamic_offset[uid] for uid in sorted(dynamic_offset)
+        }
+        path.write_text(
+            json.dumps(serialized_offsets, ensure_ascii=False, sort_keys=True),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    async def set_dynamic_offset(cls, uid: int, value: int):
+        dynamic_offset[int(uid)] = int(value)
+        await cls.save_dynamic_offsets()
+
     @classmethod
     async def get_user(cls, **kwargs):
         """获取 UP 主信息"""
@@ -277,12 +321,17 @@ class DB:
         )
 
         # 清除没有订阅的 offset
+        changed = False
         dynamic_offset_keys = set(dynamic_offset)
         dynamic_uids = set(uid_list["dynamic"]["list"])
         for uid in dynamic_offset_keys - dynamic_uids:
             del dynamic_offset[uid]
+            changed = True
         for uid in dynamic_uids - dynamic_offset_keys:
             dynamic_offset[uid] = -1
+            changed = True
+        if changed:
+            await cls.save_dynamic_offsets()
 
     async def backup(self):
         """备份数据库"""

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/libs/dynamic/web.py

@@ -47,6 +47,16 @@ def parse_web_dynamic_items(payload: dict) -> list[WebDynamicItem]:
     return parsed_items
 
 
+def parse_web_dynamic_payload(payload: dict) -> list[WebDynamicItem]:
+    if payload.get("code") != 0:
+        raise WebDynamicError(
+            payload.get("code"),
+            payload.get("message") or "unknown error",
+            payload.get("data"),
+        )
+    return parse_web_dynamic_items(payload)
+
+
 async def get_user_dynamics_web(
     uid: int,
     cookies: dict[str, str],
@@ -69,11 +79,4 @@ async def get_user_dynamics_web(
         follow_redirects=True,
     ) as client:
         response = await client.get(WEB_DYNAMIC_URL, params={"host_mid": uid})
-    payload = response.json()
-    if payload.get("code") != 0:
-        raise WebDynamicError(
-            payload.get("code"),
-            payload.get("message") or "unknown error",
-            payload.get("data"),
-        )
-    return parse_web_dynamic_items(payload)
+    return parse_web_dynamic_payload(response.json())

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/plugins/pusher/dynamic_pusher.py

@@ -19,15 +19,22 @@ from nonebot.log import logger
 from ...config import plugin_config
 from ...database import DB as db
 from ...database import dynamic_offset as offset
-from ...libs.dynamic.web import WebDynamicError, get_user_dynamics_web
+from ...libs.dynamic.web import (
+    WebDynamicError,
+    get_user_dynamics_web,
+    parse_web_dynamic_payload,
+)
 from ...utils import (
     get_bilibili_cookies,
     get_dynamic_screenshot,
+    get_user_dynamics_payload_in_browser,
     safe_send,
     scheduler,
 )
 
-RISK_CONTROL_RETRY_SECONDS = 3600
+GRPC_RISK_CONTROL_RETRY_SECONDS = 3600
+WEB_REQUEST_BANNED_RETRY_SECONDS = 300
+WEB_REQUEST_ERROR_RETRY_SECONDS = 600
 dynamic_risk_control_until = {}
 dynamic_web_fallback_until = {}
 WEB_SKIP_DYNAMIC_TYPES = {
@@ -44,6 +51,7 @@ WEB_DYNAMIC_TYPE_MESSAGES = {
     "DYNAMIC_TYPE_ARTICLE": "发布了新专栏",
     "DYNAMIC_TYPE_MUSIC": "发布了新音频",
 }
+DYNAMIC_FETCH_CONCURRENCY = 4
 
 
 async def throttle_dynamic_loop():
@@ -95,21 +103,32 @@ def should_skip_dynamic(dynamic_type, use_web_fallback: bool) -> bool:
 
 
 async def get_user_dynamics_with_web_fallback(uid: int) -> tuple[list, bool]:
+    async def fetch_web_dynamics() -> list:
+        try:
+            payload = await get_user_dynamics_payload_in_browser(uid)
+            return parse_web_dynamic_payload(payload)
+        except WebDynamicError as browser_error:
+            logger.debug(
+                f"浏览器上下文动态接口获取失败，尝试直连 Web API：{uid} "
+                f"{browser_error.code} {browser_error.msg}"
+            )
+
+        cookies = await get_bilibili_cookies()
+        if not cookies:
+            raise WebDynamicError(-1, "browser cookies unavailable")
+        return await get_user_dynamics_web(
+            uid,
+            cookies,
+            proxy=plugin_config.bililive_proxy,
+            user_agent=plugin_config.bililive_browser_ua or None,
+            timeout=plugin_config.bililive_dynamic_timeout,
+        )
+
     fallback_until = dynamic_web_fallback_until.get(uid)
     if fallback_until is not None:
         if fallback_until > monotonic():
             logger.debug(f"动态 gRPC 接口仍在风控，继续使用 Web 接口：{uid}")
-            cookies = await get_bilibili_cookies()
-            if not cookies:
-                raise WebDynamicError(-1, "browser cookies unavailable")
-            dynamics = await get_user_dynamics_web(
-                uid,
-                cookies,
-                proxy=plugin_config.bililive_proxy,
-                user_agent=plugin_config.bililive_browser_ua or None,
-                timeout=plugin_config.bililive_dynamic_timeout,
-            )
-            return dynamics, True
+            return await fetch_web_dynamics(), True
         del dynamic_web_fallback_until[uid]
 
     try:
@@ -129,36 +148,14 @@ async def get_user_dynamics_with_web_fallback(uid: int) -> tuple[list, bool]:
             f"动态 gRPC 接口触发风控，切换 Web 接口：{uid} "
             f"{e.code} {e.msg}"
         )
-        dynamic_web_fallback_until[uid] = monotonic() + RISK_CONTROL_RETRY_SECONDS
-    cookies = await get_bilibili_cookies()
-    if not cookies:
-        raise WebDynamicError(-1, "browser cookies unavailable")
-    dynamics = await get_user_dynamics_web(
-        uid,
-        cookies,
-        proxy=plugin_config.bililive_proxy,
-        user_agent=plugin_config.bililive_browser_ua or None,
-        timeout=plugin_config.bililive_dynamic_timeout,
-    )
-    return dynamics, True
+        dynamic_web_fallback_until[uid] = monotonic() + GRPC_RISK_CONTROL_RETRY_SECONDS
+    return await fetch_web_dynamics(), True
 
 
-async def dy_sched():
-    """动态推送"""
-    if not await db.wait_until_ready():
-        logger.debug("数据库尚未初始化完成，跳过本轮动态推送")
-        await throttle_dynamic_loop()
-        return
-
-    uid = await db.next_uid("dynamic")
-    if not uid:
-        # 没有订阅先暂停一秒再跳过，不然会导致 CPU 占用过高
-        await throttle_dynamic_loop()
-        return
+async def process_dynamic_uid(uid: int):
     user = await db.get_user(uid=uid)
     if user is None:
         logger.warning(f"动态推送跳过异常订阅 UID：{uid}")
-        await throttle_dynamic_loop()
         return
     name = user.name
 
@@ -186,36 +183,36 @@ async def dy_sched():
         return
     except GrpcError as e:
         logger.error(f"爬取动态失败：{e.code} {e.msg}")
-        await throttle_dynamic_loop()
         return
     except WebDynamicError as e:
-        dynamic_risk_control_until[uid] = monotonic() + RISK_CONTROL_RETRY_SECONDS
-        retry_minutes = RISK_CONTROL_RETRY_SECONDS // 60
+        retry_seconds = (
+            WEB_REQUEST_BANNED_RETRY_SECONDS
+            if e.code == -412
+            else WEB_REQUEST_ERROR_RETRY_SECONDS
+        )
+        dynamic_risk_control_until[uid] = monotonic() + retry_seconds
+        retry_minutes = max(retry_seconds // 60, 1)
         logger.warning(
             f"动态 Web 接口获取失败，{name}（{uid}）将在 "
             f"{retry_minutes} 分钟后重试：{e.code} {e.msg}"
         )
-        await throttle_dynamic_loop()
         return
 
     dynamic_risk_control_until.pop(uid, None)
 
     if not dynamics:  # 没发过动态
         if uid in offset and offset[uid] == -1:  # 不记录会导致第一次发动态不推送
-            offset[uid] = 0
+            await db.set_dynamic_offset(uid, 0)
         return
     name = get_dynamic_author_name(dynamics[0], use_web_fallback)
 
     if uid not in offset:  # 已删除
         return
     elif offset[uid] == -1:  # 第一次爬取
-        if len(dynamics) == 1:  # 只有一条动态
-            offset[uid] = get_dynamic_id(dynamics[0], use_web_fallback)
-        else:  # 第一个可能是置顶动态，但置顶也可能是最新一条，所以取前两条的最大值
-            offset[uid] = max(
-                get_dynamic_id(dynamics[0], use_web_fallback),
-                get_dynamic_id(dynamics[1], use_web_fallback),
-            )
+        await db.set_dynamic_offset(
+            uid,
+            max(get_dynamic_id(item, use_web_fallback) for item in dynamics),
+        )
         return
 
     dynamic = None
@@ -234,7 +231,7 @@ async def dy_sched():
                 return
             elif should_skip_dynamic(dynamic_type, use_web_fallback):
                 logger.debug(f"无需推送的动态 {dynamic_type}，已跳过：{url}")
-                offset[uid] = dynamic_id
+                await db.set_dynamic_offset(uid, dynamic_id)
                 return
             message = (
                 f"{name} {get_dynamic_type_message(dynamic_type, use_web_fallback)}：\n"
@@ -253,12 +250,36 @@ async def dy_sched():
                     at=bool(sets.at) and plugin_config.bililive_dynamic_at,
                 )
 
-            offset[uid] = dynamic_id
+            await db.set_dynamic_offset(uid, dynamic_id)
 
     if dynamic:
         await db.update_user(uid, name)
 
 
+async def dy_sched():
+    """动态推送"""
+    if not await db.wait_until_ready():
+        logger.debug("数据库尚未初始化完成，跳过本轮动态推送")
+        await throttle_dynamic_loop()
+        return
+
+    uids = await db.get_uid_list("dynamic")
+    if not uids:
+        # 没有订阅先暂停一秒再跳过，不然会导致 CPU 占用过高
+        await throttle_dynamic_loop()
+        return
+
+    logger.debug(f"爬取动态列表，总共 {len(uids)} 人")
+    semaphore = asyncio.Semaphore(DYNAMIC_FETCH_CONCURRENCY)
+
+    async def run_for_uid(uid: int):
+        async with semaphore:
+            await process_dynamic_uid(uid)
+
+    await asyncio.gather(*(run_for_uid(uid) for uid in uids))
+    await throttle_dynamic_loop()
+
+
 def dynamic_lisener(event):
     if hasattr(event, "job_id") and event.job_id != "dynamic_sched":
         return
@@ -280,4 +301,7 @@ else:
         "interval",
         seconds=plugin_config.bililive_dynamic_interval,
         id="dynamic_sched",
+        coalesce=True,
+        max_instances=1,
+        misfire_grace_time=5,
     )

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/utils/__init__.py

@@ -304,4 +304,8 @@ PROXIES = {"all://": plugin_config.bililive_proxy}
 require("nonebot_plugin_apscheduler")
 from nonebot_plugin_apscheduler import scheduler  # noqa
 
-from .browser import get_bilibili_cookies, get_dynamic_screenshot  # noqa
+from .browser import (  # noqa
+    get_bilibili_cookies,
+    get_dynamic_screenshot,
+    get_user_dynamics_payload_in_browser,
+)

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/utils/browser.py

@@ -16,6 +16,7 @@ from .fonts_provider import fill_font
 
 _browser: BrowserContext | None = None
 mobile_js = Path(__file__).parent.joinpath("mobile.js")
+WEB_DYNAMIC_URL = "https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space"
 
 
 async def init_browser(proxy=plugin_config.bililive_proxy, **kwargs) -> BrowserContext:
@@ -72,6 +73,32 @@ async def get_bilibili_cookies() -> dict[str, str]:
     return {cookie["name"]: cookie["value"] for cookie in cookies}
 
 
+async def get_user_dynamics_payload_in_browser(uid: int) -> dict:
+    browser = await get_browser()
+    page = await browser.new_page()
+    try:
+        await page.goto(
+            "https://www.bilibili.com/",
+            wait_until="domcontentloaded",
+            timeout=plugin_config.bililive_dynamic_timeout * 1000,
+        )
+        return await page.evaluate(
+            """async ({ url, uid }) => {
+                const response = await fetch(`${url}?host_mid=${uid}`, {
+                    credentials: 'include',
+                    headers: {
+                        accept: 'application/json, text/plain, */*',
+                    },
+                });
+                return await response.json();
+            }""",
+            {"url": WEB_DYNAMIC_URL, "uid": str(uid)},
+        )
+    finally:
+        with contextlib.suppress(Exception):
+            await page.close()
+
+
 async def get_dynamic_screenshot(
     dynamic_id,
     style=plugin_config.bililive_screenshot_style,

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/utils/mobile.js

@@ -183,13 +183,13 @@ function setFont(font = "", fontSource = "local") {
     // 将字体样式设置到 div#app 上
     const appDom = document.querySelector("#app");
     const emojiFont = emojiFontList.join(",");
+    const fallbackFonts = needLoadFontList.map(fontObject => fontObject.fontFamily).join(",");
     if (appDom) {
         // 动态加字体, 并给与默认值 sans-serif
         if (fontSource === "system") {
-            appDom.style.fontFamily = font + "," + emojiFont + ",sans-serif";
+            appDom.style.fontFamily = [font, fallbackFonts, emojiFont, "sans-serif"].filter(Boolean).join(",");
         } else {
-            const needLoadFont = needLoadFontList.reduce((defaultString, fontObject) => defaultString + fontObject.fontFamily + ",", "");
-            appDom.style.fontFamily = needLoadFont + emojiFont + ",sans-serif";
+            appDom.style.fontFamily = [fallbackFonts, emojiFont, "sans-serif"].filter(Boolean).join(",");
         };
         appDom.style.overflowWrap = "break-word";
     }

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/bililive/version.py

@@ -1,4 +1,4 @@
 from packaging.version import Version
 
-__version__ = "2.0.4"
+__version__ = "2.0.6"
 VERSION = Version(__version__)

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "msvc-runtime>=14.34.31931; sys_platform == \"win32\"",
 ]
 dynamic = []
-version = "2.0.4"
+version = "2.0.6"
 
 [project.license]
 text = "AGPL-3.0-or-later"

{nonebot_plugin_bililive-2.0.4 → nonebot_plugin_bililive-2.0.6}/tests/test_maintenance.py

@@ -1,6 +1,8 @@
 import sys
+import tempfile
 import unittest
 from importlib import import_module
+from pathlib import Path
 from types import ModuleType, SimpleNamespace
 from unittest.mock import AsyncMock, patch
 
@@ -18,7 +20,20 @@ class DummyDriver:
 
 
 fake_apscheduler = ModuleType("nonebot_plugin_apscheduler")
-fake_apscheduler.scheduler = SimpleNamespace()
+
+
+class DummyScheduler:
+    def add_listener(self, *args, **kwargs):
+        return None
+
+    def add_job(self, *args, **kwargs):
+        return None
+
+    def get_job(self, *args, **kwargs):
+        return None
+
+
+fake_apscheduler.scheduler = DummyScheduler()
 
 
 with patch("nonebot.get_driver", return_value=DummyDriver()), patch(
@@ -129,6 +144,18 @@ class WebDynamicTests(unittest.TestCase):
 
         self.assertEqual(web_dynamic.parse_web_dynamic_items(payload), [])
 
+    def test_parse_web_dynamic_payload_raises_for_error_code(self):
+        payload = {
+            "code": -412,
+            "message": "request was banned",
+            "data": None,
+        }
+
+        with self.assertRaises(web_dynamic.WebDynamicError) as context:
+            web_dynamic.parse_web_dynamic_payload(payload)
+
+        self.assertEqual(context.exception.code, -412)
+
 
 class DBPermissionTests(unittest.IsolatedAsyncioTestCase):
     async def test_db_init_enables_global_fallback(self):
@@ -157,6 +184,23 @@ class DBPermissionTests(unittest.IsolatedAsyncioTestCase):
 
         self.assertFalse(ready)
 
+    async def test_dynamic_offsets_are_persisted_and_restored(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            offset_path = Path(tmpdir) / "dynamic_offset.json"
+            db_module.dynamic_offset.clear()
+            db_module.dynamic_offset[123] = 456
+
+            with patch.object(db_module, "get_path", return_value=str(offset_path)):
+                await DB.save_dynamic_offsets()
+
+                db_module.dynamic_offset.clear()
+                db_module.dynamic_offset[123] = -1
+                db_module.dynamic_offset[789] = -1
+                await DB.load_dynamic_offsets()
+
+            self.assertEqual(db_module.dynamic_offset[123], 456)
+            self.assertEqual(db_module.dynamic_offset[789], -1)
+
     async def test_set_permission_creates_group_when_missing(self):
         with (
             patch.object(DB, "get_group", new=AsyncMock(return_value=None)),