nonebot-plugin-bililive 2.0.5__tar.gz → 2.0.6__tar.gz

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nonebot-plugin-bililive
-Version: 2.0.5
+Version: 2.0.6
 Summary: Push bilibili dynamics and live notifications to QQ with NoneBot2.
 Keywords: nonebot,nonebot2,nonebot-plugin,qqbot,bilibili
 Author-Email: SK-415 <2967923486@qq.com>

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/bililive/libs/dynamic/web.py

@@ -47,6 +47,16 @@ def parse_web_dynamic_items(payload: dict) -> list[WebDynamicItem]:
     return parsed_items
 
 
+def parse_web_dynamic_payload(payload: dict) -> list[WebDynamicItem]:
+    if payload.get("code") != 0:
+        raise WebDynamicError(
+            payload.get("code"),
+            payload.get("message") or "unknown error",
+            payload.get("data"),
+        )
+    return parse_web_dynamic_items(payload)
+
+
 async def get_user_dynamics_web(
     uid: int,
     cookies: dict[str, str],
@@ -69,11 +79,4 @@ async def get_user_dynamics_web(
         follow_redirects=True,
     ) as client:
         response = await client.get(WEB_DYNAMIC_URL, params={"host_mid": uid})
-    payload = response.json()
-    if payload.get("code") != 0:
-        raise WebDynamicError(
-            payload.get("code"),
-            payload.get("message") or "unknown error",
-            payload.get("data"),
-        )
-    return parse_web_dynamic_items(payload)
+    return parse_web_dynamic_payload(response.json())

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/bililive/plugins/pusher/dynamic_pusher.py

@@ -19,15 +19,22 @@ from nonebot.log import logger
 from ...config import plugin_config
 from ...database import DB as db
 from ...database import dynamic_offset as offset
-from ...libs.dynamic.web import WebDynamicError, get_user_dynamics_web
+from ...libs.dynamic.web import (
+    WebDynamicError,
+    get_user_dynamics_web,
+    parse_web_dynamic_payload,
+)
 from ...utils import (
     get_bilibili_cookies,
     get_dynamic_screenshot,
+    get_user_dynamics_payload_in_browser,
     safe_send,
     scheduler,
 )
 
-RISK_CONTROL_RETRY_SECONDS = 3600
+GRPC_RISK_CONTROL_RETRY_SECONDS = 3600
+WEB_REQUEST_BANNED_RETRY_SECONDS = 300
+WEB_REQUEST_ERROR_RETRY_SECONDS = 600
 dynamic_risk_control_until = {}
 dynamic_web_fallback_until = {}
 WEB_SKIP_DYNAMIC_TYPES = {
@@ -44,6 +51,7 @@ WEB_DYNAMIC_TYPE_MESSAGES = {
     "DYNAMIC_TYPE_ARTICLE": "发布了新专栏",
     "DYNAMIC_TYPE_MUSIC": "发布了新音频",
 }
+DYNAMIC_FETCH_CONCURRENCY = 4
 
 
 async def throttle_dynamic_loop():
@@ -95,21 +103,32 @@ def should_skip_dynamic(dynamic_type, use_web_fallback: bool) -> bool:
 
 
 async def get_user_dynamics_with_web_fallback(uid: int) -> tuple[list, bool]:
+    async def fetch_web_dynamics() -> list:
+        try:
+            payload = await get_user_dynamics_payload_in_browser(uid)
+            return parse_web_dynamic_payload(payload)
+        except WebDynamicError as browser_error:
+            logger.debug(
+                f"浏览器上下文动态接口获取失败，尝试直连 Web API：{uid} "
+                f"{browser_error.code} {browser_error.msg}"
+            )
+
+        cookies = await get_bilibili_cookies()
+        if not cookies:
+            raise WebDynamicError(-1, "browser cookies unavailable")
+        return await get_user_dynamics_web(
+            uid,
+            cookies,
+            proxy=plugin_config.bililive_proxy,
+            user_agent=plugin_config.bililive_browser_ua or None,
+            timeout=plugin_config.bililive_dynamic_timeout,
+        )
+
     fallback_until = dynamic_web_fallback_until.get(uid)
     if fallback_until is not None:
         if fallback_until > monotonic():
             logger.debug(f"动态 gRPC 接口仍在风控，继续使用 Web 接口：{uid}")
-            cookies = await get_bilibili_cookies()
-            if not cookies:
-                raise WebDynamicError(-1, "browser cookies unavailable")
-            dynamics = await get_user_dynamics_web(
-                uid,
-                cookies,
-                proxy=plugin_config.bililive_proxy,
-                user_agent=plugin_config.bililive_browser_ua or None,
-                timeout=plugin_config.bililive_dynamic_timeout,
-            )
-            return dynamics, True
+            return await fetch_web_dynamics(), True
         del dynamic_web_fallback_until[uid]
 
     try:
@@ -129,36 +148,14 @@ async def get_user_dynamics_with_web_fallback(uid: int) -> tuple[list, bool]:
             f"动态 gRPC 接口触发风控，切换 Web 接口：{uid} "
             f"{e.code} {e.msg}"
         )
-        dynamic_web_fallback_until[uid] = monotonic() + RISK_CONTROL_RETRY_SECONDS
-    cookies = await get_bilibili_cookies()
-    if not cookies:
-        raise WebDynamicError(-1, "browser cookies unavailable")
-    dynamics = await get_user_dynamics_web(
-        uid,
-        cookies,
-        proxy=plugin_config.bililive_proxy,
-        user_agent=plugin_config.bililive_browser_ua or None,
-        timeout=plugin_config.bililive_dynamic_timeout,
-    )
-    return dynamics, True
-
+        dynamic_web_fallback_until[uid] = monotonic() + GRPC_RISK_CONTROL_RETRY_SECONDS
+    return await fetch_web_dynamics(), True
 
-async def dy_sched():
-    """动态推送"""
-    if not await db.wait_until_ready():
-        logger.debug("数据库尚未初始化完成，跳过本轮动态推送")
-        await throttle_dynamic_loop()
-        return
 
-    uid = await db.next_uid("dynamic")
-    if not uid:
-        # 没有订阅先暂停一秒再跳过，不然会导致 CPU 占用过高
-        await throttle_dynamic_loop()
-        return
+async def process_dynamic_uid(uid: int):
     user = await db.get_user(uid=uid)
     if user is None:
         logger.warning(f"动态推送跳过异常订阅 UID：{uid}")
-        await throttle_dynamic_loop()
         return
     name = user.name
 
@@ -186,16 +183,19 @@ async def dy_sched():
         return
     except GrpcError as e:
         logger.error(f"爬取动态失败：{e.code} {e.msg}")
-        await throttle_dynamic_loop()
         return
     except WebDynamicError as e:
-        dynamic_risk_control_until[uid] = monotonic() + RISK_CONTROL_RETRY_SECONDS
-        retry_minutes = RISK_CONTROL_RETRY_SECONDS // 60
+        retry_seconds = (
+            WEB_REQUEST_BANNED_RETRY_SECONDS
+            if e.code == -412
+            else WEB_REQUEST_ERROR_RETRY_SECONDS
+        )
+        dynamic_risk_control_until[uid] = monotonic() + retry_seconds
+        retry_minutes = max(retry_seconds // 60, 1)
         logger.warning(
             f"动态 Web 接口获取失败，{name}（{uid}）将在 "
             f"{retry_minutes} 分钟后重试：{e.code} {e.msg}"
         )
-        await throttle_dynamic_loop()
         return
 
     dynamic_risk_control_until.pop(uid, None)
@@ -256,6 +256,30 @@ async def dy_sched():
         await db.update_user(uid, name)
 
 
+async def dy_sched():
+    """动态推送"""
+    if not await db.wait_until_ready():
+        logger.debug("数据库尚未初始化完成，跳过本轮动态推送")
+        await throttle_dynamic_loop()
+        return
+
+    uids = await db.get_uid_list("dynamic")
+    if not uids:
+        # 没有订阅先暂停一秒再跳过，不然会导致 CPU 占用过高
+        await throttle_dynamic_loop()
+        return
+
+    logger.debug(f"爬取动态列表，总共 {len(uids)} 人")
+    semaphore = asyncio.Semaphore(DYNAMIC_FETCH_CONCURRENCY)
+
+    async def run_for_uid(uid: int):
+        async with semaphore:
+            await process_dynamic_uid(uid)
+
+    await asyncio.gather(*(run_for_uid(uid) for uid in uids))
+    await throttle_dynamic_loop()
+
+
 def dynamic_lisener(event):
     if hasattr(event, "job_id") and event.job_id != "dynamic_sched":
         return
@@ -277,4 +301,7 @@ else:
         "interval",
         seconds=plugin_config.bililive_dynamic_interval,
         id="dynamic_sched",
+        coalesce=True,
+        max_instances=1,
+        misfire_grace_time=5,
     )

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/bililive/utils/__init__.py

@@ -304,4 +304,8 @@ PROXIES = {"all://": plugin_config.bililive_proxy}
 require("nonebot_plugin_apscheduler")
 from nonebot_plugin_apscheduler import scheduler  # noqa
 
-from .browser import get_bilibili_cookies, get_dynamic_screenshot  # noqa
+from .browser import (  # noqa
+    get_bilibili_cookies,
+    get_dynamic_screenshot,
+    get_user_dynamics_payload_in_browser,
+)

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/bililive/utils/browser.py

@@ -16,6 +16,7 @@ from .fonts_provider import fill_font
 
 _browser: BrowserContext | None = None
 mobile_js = Path(__file__).parent.joinpath("mobile.js")
+WEB_DYNAMIC_URL = "https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space"
 
 
 async def init_browser(proxy=plugin_config.bililive_proxy, **kwargs) -> BrowserContext:
@@ -72,6 +73,32 @@ async def get_bilibili_cookies() -> dict[str, str]:
     return {cookie["name"]: cookie["value"] for cookie in cookies}
 
 
+async def get_user_dynamics_payload_in_browser(uid: int) -> dict:
+    browser = await get_browser()
+    page = await browser.new_page()
+    try:
+        await page.goto(
+            "https://www.bilibili.com/",
+            wait_until="domcontentloaded",
+            timeout=plugin_config.bililive_dynamic_timeout * 1000,
+        )
+        return await page.evaluate(
+            """async ({ url, uid }) => {
+                const response = await fetch(`${url}?host_mid=${uid}`, {
+                    credentials: 'include',
+                    headers: {
+                        accept: 'application/json, text/plain, */*',
+                    },
+                });
+                return await response.json();
+            }""",
+            {"url": WEB_DYNAMIC_URL, "uid": str(uid)},
+        )
+    finally:
+        with contextlib.suppress(Exception):
+            await page.close()
+
+
 async def get_dynamic_screenshot(
     dynamic_id,
     style=plugin_config.bililive_screenshot_style,

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/bililive/version.py

@@ -1,4 +1,4 @@
 from packaging.version import Version
 
-__version__ = "2.0.5"
+__version__ = "2.0.6"
 VERSION = Version(__version__)

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "msvc-runtime>=14.34.31931; sys_platform == \"win32\"",
 ]
 dynamic = []
-version = "2.0.5"
+version = "2.0.6"
 
 [project.license]
 text = "AGPL-3.0-or-later"

{nonebot_plugin_bililive-2.0.5 → nonebot_plugin_bililive-2.0.6}/tests/test_maintenance.py

@@ -20,7 +20,20 @@ class DummyDriver:
 
 
 fake_apscheduler = ModuleType("nonebot_plugin_apscheduler")
-fake_apscheduler.scheduler = SimpleNamespace()
+
+
+class DummyScheduler:
+    def add_listener(self, *args, **kwargs):
+        return None
+
+    def add_job(self, *args, **kwargs):
+        return None
+
+    def get_job(self, *args, **kwargs):
+        return None
+
+
+fake_apscheduler.scheduler = DummyScheduler()
 
 
 with patch("nonebot.get_driver", return_value=DummyDriver()), patch(
@@ -131,6 +144,18 @@ class WebDynamicTests(unittest.TestCase):
 
         self.assertEqual(web_dynamic.parse_web_dynamic_items(payload), [])
 
+    def test_parse_web_dynamic_payload_raises_for_error_code(self):
+        payload = {
+            "code": -412,
+            "message": "request was banned",
+            "data": None,
+        }
+
+        with self.assertRaises(web_dynamic.WebDynamicError) as context:
+            web_dynamic.parse_web_dynamic_payload(payload)
+
+        self.assertEqual(context.exception.code, -412)
+
 
 class DBPermissionTests(unittest.IsolatedAsyncioTestCase):
     async def test_db_init_enables_global_fallback(self):