aidev-bkplugin 2.2.4__tar.gz → 2.4.0__tar.gz

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: aidev-bkplugin
-Version: 2.2.4
+Version: 2.4.0
 Summary: AiDev Agent Bkplugin builtin functions
 Requires-Python: <3.12,>=3.11
-Requires-Dist: aidev-agent>=2.2.4
+Requires-Dist: aidev-agent>=2.4.0
 Requires-Dist: blueapps<5.0,>=4.15.1
 Requires-Dist: django<5.0,>=4.2.28
 Provides-Extra: opentelemetry

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/aidev_bkplugin/apps.py

@@ -8,6 +8,7 @@ from aidev_agent.utils.module_loading import import_string
 from django.apps import AppConfig
 from django.conf import settings
 
+from aidev_bkplugin.celery_runtime import install_celery_exit_on_broker_loss
 from aidev_bkplugin.services.metric_runtime import set_metric_service
 
 try:
@@ -224,6 +225,9 @@ class AgentConfig(AppConfig):
         if custom_resource_manager:
             resource_manager.replace_defaults(import_string(custom_resource_manager)())
 
+        # Celery worker 断连退出，避免 consumers=0 假活。
+        install_celery_exit_on_broker_loss()
+
         # 初始化 OpenTelemetry
         init_bk_aidev_agent_otel()
 

aidev_bkplugin-2.4.0/aidev_bkplugin/celery_runtime.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""Celery worker 与 broker 断连后的退出策略。
+
+默认 Celery 会无限重连。重连失败或 consumers 未恢复时进程仍活着，
+队列 ``consumers=0``，任务堆积，表现为服务假死。
+连上之后禁止重连，让 worker 以非 0 退出，由 PaaS 拉起新进程。
+启动阶段仍允许重试，避免 broker 尚未就绪时直接起不来。
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Django settings / Celery 5 namespace 两套名字都写，兼容 blueapps 是否带 CELERY_ 前缀。
+BROKER_LOSS_EXIT_SETTINGS = {
+    "BROKER_CONNECTION_RETRY": False,
+    "BROKER_CONNECTION_RETRY_ON_STARTUP": True,
+    "WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS": True,
+    "CELERY_BROKER_CONNECTION_RETRY": False,
+    "CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP": True,
+    "CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS": True,
+}
+
+
+def is_celery_worker(argv: list[str] | None = None) -> bool:
+    """判断当前进程是否为 Celery worker（不含 beat）。"""
+    args = list(argv if argv is not None else sys.argv)
+    if not args:
+        return False
+    program = Path(args[0]).name.lower()
+    rest = args[1:]
+    if program in {"celery", "celery.exe"}:
+        return "worker" in rest
+    if rest and rest[0] == "celery":
+        return "worker" in rest[1:]
+    return False
+
+
+def apply_celery_exit_on_broker_loss(app: Any) -> None:
+    """Disable post-connect broker reconnect so a lost consumer shuts the worker down."""
+    conf = app.conf
+    conf.broker_connection_retry = False
+    conf.broker_connection_retry_on_startup = True
+    conf.worker_cancel_long_running_tasks_on_connection_loss = True
+
+
+def install_celery_exit_on_broker_loss(argv: list[str] | None = None) -> bool:
+    """Worker 进程里关掉连上之后的 broker 重连。非 worker 或未安装 Celery 时跳过。"""
+    if not is_celery_worker(argv):
+        return False
+    try:
+        from celery import current_app
+    except ImportError:
+        logger.warning("[aidev_bkplugin] celery is not installed; skip broker-loss exit hook")
+        return False
+
+    apps = [current_app]
+    try:
+        from blueapps.core.celery import app as blueapps_app
+    except ImportError:
+        blueapps_app = None
+    if blueapps_app is not None and blueapps_app is not current_app:
+        apps.append(blueapps_app)
+
+    for app in apps:
+        apply_celery_exit_on_broker_loss(app)
+    logger.warning(
+        "[aidev_bkplugin] Celery broker reconnect after connect is disabled; "
+        "worker will exit on connection loss so the platform can restart it"
+    )
+    return True

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/aidev_bkplugin/settings.py

@@ -37,6 +37,15 @@ RUN_VER = "ieod" if os.environ.get("BKPAAS_ENGINE_REGION", "default") == "ieod"
 # OAuth 认证配置
 BKAUTH_BACKEND_TYPE = "bk_ticket" if RUN_VER == "ieod" else "bk_token"
 
+# Celery worker 连上 broker 后禁止假活重连；断连以非 0 退出，由 PaaS 拉起新 worker。
+# 启动阶段仍允许重试。Django 无 namespace 与 CELERY_ 前缀两套都写，兼容 blueapps。
+BROKER_CONNECTION_RETRY = False
+BROKER_CONNECTION_RETRY_ON_STARTUP = True
+WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True
+CELERY_BROKER_CONNECTION_RETRY = False
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
+CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True
+
 # 仅社区版需要配置，其它版本已由开发框架处理
 if BKAUTH_BACKEND_TYPE == 'bk_token':
     OAUTH_COOKIES_PARAMS = {"bk_token": "bk_token", "bk_uid": "bk_uid"}

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/aidev_bkplugin/views/session.py

@@ -11,9 +11,11 @@ from aidev_agent.services.sandbox_pv_files import (
     SandboxFileInvalidRequestError,
     SandboxFileNotFoundError,
     SandboxPvFileService,
+    SandboxVolumeNotFoundError,
     fill_user_image_urls,
     iter_user_images_missing_url,
     validate_session_upload_files,
+    validate_session_upload_stats,
 )
 from bkapi_client_core.exceptions import HTTPResponseError
 from blueapps.core.exceptions import ClientBlueException, ResourceNotFound, ServerBlueException
@@ -236,11 +238,10 @@ class ChatSessionViewSet(PluginViewSet):
     def pv_files(self, request, pk, **kwargs):
         self._check_session_owner(request, pk, require_access=False)
         svc = self._make_pv_file_service(request)
-        params = request.query_params
         try:
             data = svc.list_files(
                 session_code=pk,
-                path=params.get("path", ""),
+                path=request.query_params.get("path"),
                 since=None,
                 until=None,
             )
@@ -248,14 +249,29 @@ class ChatSessionViewSet(PluginViewSet):
             self._raise_pv_exc(exc)
         return Response(data=data)
 
+    @pv_files.mapping.delete
+    def delete_pv_file(self, request, pk, **kwargs):
+        self._check_session_owner(request, pk, require_access=True)
+        try:
+            self._make_pv_file_service(request).delete_file(
+                session_code=pk, path=request.query_params.get("path")
+            )
+        # 幂等只覆盖「文件不存在」；会话根本没有 PV 是真错误，不能当删除成功返回 200
+        except SandboxVolumeNotFoundError as exc:
+            self._raise_pv_exc(exc)
+        except SandboxFileNotFoundError:
+            pass
+        except SandboxFileError as exc:
+            self._raise_pv_exc(exc)
+        return Response()
+
     @action(["GET"], url_path="pv_files/stat", detail=True)
     def pv_files_stat(self, request, pk, **kwargs):
         self._check_session_owner(request, pk, require_access=False)
-        path = request.query_params.get("path", "")
-        if not path:
-            raise ClientBlueException(message="path is required")
         try:
-            data = self._make_pv_file_service(request).stat_file(session_code=pk, path=path)
+            data = self._make_pv_file_service(request).stat_file(
+                session_code=pk, path=request.query_params.get("path")
+            )
         except SandboxFileError as exc:
             self._raise_pv_exc(exc)
         return Response(data=data)
@@ -263,13 +279,10 @@ class ChatSessionViewSet(PluginViewSet):
     @action(["GET"], url_path="pv_files/preview", detail=True)
     def pv_files_preview(self, request, pk, **kwargs):
         self._check_session_owner(request, pk, require_access=False)
-        path = request.query_params.get("path", "")
-        if not path:
-            raise ClientBlueException(message="path is required")
         max_bytes = _parse_positive_int(request.query_params.get("max_bytes"), 65536)
         try:
             content, truncated = self._make_pv_file_service(request).preview_file(
-                session_code=pk, path=path, max_bytes=max_bytes
+                session_code=pk, path=request.query_params.get("path"), max_bytes=max_bytes
             )
         except SandboxFileError as exc:
             self._raise_pv_exc(exc)
@@ -280,13 +293,10 @@ class ChatSessionViewSet(PluginViewSet):
     @action(["GET"], url_path="pv_files/download_url", detail=True)
     def pv_files_download_url(self, request, pk, **kwargs):
         self._check_session_owner(request, pk, require_access=False)
-        path = request.query_params.get("path", "")
-        if not path:
-            raise ClientBlueException(message="path is required")
         expires_in = _parse_positive_int(request.query_params.get("expires_in"), 600)
         try:
             data = self._make_pv_file_service(request).get_download_url(
-                session_code=pk, path=path, expires_in=expires_in
+                session_code=pk, path=request.query_params.get("path"), expires_in=expires_in
             )
         except SandboxFileError as exc:
             self._raise_pv_exc(exc)
@@ -302,15 +312,16 @@ class ChatSessionViewSet(PluginViewSet):
         """批量上传文件到会话 PV。"""
         self._check_session_owner(request, pk, require_access=True)
         uploaded_files = request.FILES.getlist("files")
-        files = [
-            {
-                "name": upload_file.name,
-                "content": upload_file.read(),
-                "mime_type": upload_file.content_type or "application/octet-stream",
-            }
-            for upload_file in uploaded_files
-        ]
         try:
+            validate_session_upload_stats(uploaded_files)
+            files = [
+                {
+                    "name": upload_file.name,
+                    "content": upload_file.read(),
+                    "mime_type": upload_file.content_type or "application/octet-stream",
+                }
+                for upload_file in uploaded_files
+            ]
             validate_session_upload_files(files)
         except SandboxFileError as exc:
             self._raise_pv_exc(exc)

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "aidev-bkplugin"
-version = "2.2.4"
+version = "2.4.0"
 description = "AiDev Agent Bkplugin builtin functions"
 readme = "readme.md"
 requires-python = "<3.12,>=3.11"
 dependencies = [
-    "aidev-agent>=2.2.4",
+    "aidev-agent>=2.4.0",
     "blueapps>=4.15.1, <5.0",
     "django>=4.2.28,<5.0",
 ]

aidev_bkplugin-2.4.0/tests/test_celery_runtime.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+from types import ModuleType, SimpleNamespace
+
+import pytest
+from aidev_bkplugin import settings as plugin_settings
+from aidev_bkplugin.celery_runtime import (
+    BROKER_LOSS_EXIT_SETTINGS,
+    apply_celery_exit_on_broker_loss,
+    install_celery_exit_on_broker_loss,
+    is_celery_worker,
+)
+
+
+@pytest.mark.parametrize(
+    ("argv", "expected"),
+    [
+        (["celery", "-A", "blueapps.core.celery", "worker", "-P", "threads"], True),
+        (["/usr/local/bin/celery", "-A", "test_app", "worker"], True),
+        (["manage.py", "celery", "worker", "-B"], True),
+        (["bin/manage.py", "celery", "-A", "test_app", "worker"], True),
+        (["celery", "-A", "blueapps.core.celery", "beat"], False),
+        (["manage.py", "celery", "beat", "-l", "info"], False),
+        (["gunicorn", "wsgi:application"], False),
+        (["manage.py", "runserver"], False),
+        ([], False),
+    ],
+)
+def test_is_celery_worker_detects_worker_only(argv, expected):
+    assert is_celery_worker(argv) is expected
+
+
+def test_apply_celery_exit_on_broker_loss_disables_reconnect_after_connect():
+    app = SimpleNamespace(conf=SimpleNamespace())
+
+    apply_celery_exit_on_broker_loss(app)
+
+    assert app.conf.broker_connection_retry is False
+    assert app.conf.broker_connection_retry_on_startup is True
+    assert app.conf.worker_cancel_long_running_tasks_on_connection_loss is True
+
+
+def test_install_applies_when_celery_is_available(mocker):
+    app = SimpleNamespace(conf=SimpleNamespace())
+    celery_mod = ModuleType("celery")
+    celery_mod.current_app = app
+    mocker.patch.dict("sys.modules", {"celery": celery_mod})
+
+    assert install_celery_exit_on_broker_loss(["celery", "-A", "app", "worker"]) is True
+    assert app.conf.broker_connection_retry is False
+    assert app.conf.broker_connection_retry_on_startup is True
+
+
+def test_plugin_settings_disable_broker_reconnect_after_connect():
+    for name, value in BROKER_LOSS_EXIT_SETTINGS.items():
+        assert getattr(plugin_settings, name) is value
+
+
+def test_install_skips_non_worker_process():
+    assert install_celery_exit_on_broker_loss(["gunicorn", "wsgi:application"]) is False

{aidev_bkplugin-2.2.4 → aidev_bkplugin-2.4.0}/tests/views/test_session_pv_files_views.py

@@ -3,7 +3,7 @@
 
 覆盖点：
 - 构造沙箱文件 Service 时正确注入 PluginResourceManager + executor_info
-- 5 个 action 参数透传（GET list / stat / preview / download_url / upload）
+- 6 个 action 参数透传（GET list / DELETE / stat / preview / download_url / upload）
 - 上传会话归属校验与沙箱文件异常映射
 - 沙箱文件异常 → blueapps 异常映射（404 / 400 / 500）
 - preview 返回 HttpResponse(text/plain) + X-Truncated 头透传
@@ -60,6 +60,7 @@ from aidev_agent.services.sandbox_pv_files import (  # noqa: E402
     SandboxFileInvalidRequestError,
     SandboxFileNotFoundError,
     SandboxFileServerError,
+    SandboxVolumeNotFoundError,
 )
 from aidev_bkplugin.views import session as session_mod  # noqa: E402
 
@@ -203,6 +204,7 @@ class TestPvFilesGet:
         instance.list_files.return_value = {"count": 1, "results": [{"path": "a.txt"}]}
         response = view.pv_files(_request({"path": "sub/"}), pk="s1")
         assert response.data == {"count": 1, "results": [{"path": "a.txt"}]}
+        # 视图只透传原始入参，归一化由 Service 负责
         instance.list_files.assert_called_once_with(session_code="s1", path="sub/", since=None, until=None)
 
     def test_list_translates_not_found(self, view, mock_svc):
@@ -282,6 +284,53 @@ class TestCheckSessionOwner:
         fake_client.api.retrieve_chat_session.assert_not_called()
 
 
+# ---------------------------------------------------------------------------
+# pv_files DELETE
+# ---------------------------------------------------------------------------
+
+
+class TestPvFilesDelete:
+    def test_delete_forwards_path_and_returns_ok(self, view, mock_svc):
+        instance, _ = mock_svc
+        instance.delete_file.return_value = None
+        response = view.delete_pv_file(_request({"path": "files/a.txt"}, method="DELETE"), pk="s1")
+        assert response.status_code == 200
+        instance.delete_file.assert_called_once_with(session_code="s1", path="files/a.txt")
+        fake_client.api.retrieve_chat_session.assert_called_once()
+
+    def test_delete_invalid_path_is_not_idempotent(self, view, mock_svc):
+        """非法路径由 Service 判定；幂等分支不能把它一起吞成 200。"""
+        from blueapps.core.exceptions import ClientBlueException
+
+        instance, _ = mock_svc
+        instance.delete_file.side_effect = SandboxFileInvalidArgumentError("invalid path")
+        with pytest.raises(ClientBlueException) as excinfo:
+            view.delete_pv_file(_request({"path": "files/../secret.txt"}, method="DELETE"), pk="s1")
+        assert excinfo.value.STATUS_CODE == 400
+
+    def test_delete_not_found_is_idempotent(self, view, mock_svc):
+        instance, _ = mock_svc
+        instance.delete_file.side_effect = SandboxFileNotFoundError("nf")
+        response = view.delete_pv_file(_request({"path": "files/gone.txt"}, method="DELETE"), pk="s1")
+        assert response.status_code == 200
+
+    def test_delete_missing_volume_is_not_idempotent(self, view, mock_svc):
+        """会话没有 PV 不属于「文件已删掉」，不能被幂等吞成 200。"""
+        from blueapps.core.exceptions import ResourceNotFound
+
+        instance, _ = mock_svc
+        instance.delete_file.side_effect = SandboxVolumeNotFoundError("session s1 未初始化沙箱 PersistentVolume")
+        with pytest.raises(ResourceNotFound):
+            view.delete_pv_file(_request({"path": "files/a.txt"}, method="DELETE"), pk="s1")
+
+    def test_delete_checks_owner(self, view, mock_svc):
+        from blueapps.core.exceptions import ClientBlueException
+
+        fake_client.api.retrieve_chat_session.side_effect = TestCheckSessionOwner._make_http_error(403)
+        with pytest.raises(ClientBlueException):
+            view.delete_pv_file(_request({"path": "files/a.txt"}, method="DELETE"), pk="s1")
+
+
 # ---------------------------------------------------------------------------
 # pv_files_stat
 # ---------------------------------------------------------------------------
@@ -295,12 +344,6 @@ class TestPvFilesStat:
         assert response.data == {"exists": True, "size": 42}
         instance.stat_file.assert_called_once_with(session_code="s1", path="x.txt")
 
-    def test_stat_requires_path(self, view, mock_svc):
-        from blueapps.core.exceptions import ClientBlueException
-
-        with pytest.raises(ClientBlueException):
-            view.pv_files_stat(_request({}), pk="s1")
-
     def test_stat_translates_invalid_arg(self, view, mock_svc):
         from blueapps.core.exceptions import ClientBlueException
 
@@ -354,12 +397,6 @@ class TestPvFilesPreview:
             view.pv_files_preview(_request({"path": "x.bin"}), pk="s1")
         assert excinfo.value.STATUS_CODE == 400
 
-    def test_preview_requires_path(self, view, mock_svc):
-        from blueapps.core.exceptions import ClientBlueException
-
-        with pytest.raises(ClientBlueException):
-            view.pv_files_preview(_request({}), pk="s1")
-
 
 # ---------------------------------------------------------------------------
 # pv_files_download_url
@@ -389,12 +426,6 @@ class TestPvFilesDownloadUrl:
             view.pv_files_download_url(_request({"path": "x.txt"}), pk="s1")
         assert excinfo.value.STATUS_CODE == 500
 
-    def test_download_url_requires_path(self, view, mock_svc):
-        from blueapps.core.exceptions import ClientBlueException
-
-        with pytest.raises(ClientBlueException):
-            view.pv_files_download_url(_request({}), pk="s1")
-
 
 class TestPvFilesUpload:
     def test_upload_forwards_files_to_local_service(self, view, mock_svc):
@@ -432,6 +463,50 @@ class TestPvFilesUpload:
         instance.upload_files.assert_not_called()
         session_mod.PluginResourceManager.return_value.get_client.return_value.api.retrieve_latest_skill_version_image.assert_not_called()
 
+    def test_upload_rejects_oversized_file_without_reading(self, view, mock_svc):
+        from aidev_agent.services.sandbox_pv_files import MAX_SESSION_UPLOAD_FILE_SIZE
+        from blueapps.core.exceptions import ClientBlueException
+
+        instance, _ = mock_svc
+        uploaded_file = SimpleNamespace(
+            name="big.txt",
+            size=MAX_SESSION_UPLOAD_FILE_SIZE + 1,
+            content_type="text/plain",
+            read=MagicMock(side_effect=AssertionError("oversized file must not be read")),
+        )
+
+        with pytest.raises(ClientBlueException) as excinfo:
+            view.pv_files_upload(_request(method="POST", files=[uploaded_file]), pk="s1")
+
+        assert str(MAX_SESSION_UPLOAD_FILE_SIZE) in excinfo.value.message
+        assert excinfo.value.STATUS_CODE == 400
+        uploaded_file.read.assert_not_called()
+        instance.upload_files.assert_not_called()
+
+    def test_upload_rejects_too_many_files_without_reading(self, view, mock_svc):
+        from aidev_agent.services.sandbox_pv_files import MAX_SESSION_UPLOAD_FILES
+        from blueapps.core.exceptions import ClientBlueException
+
+        instance, _ = mock_svc
+        uploaded_files = [
+            SimpleNamespace(
+                name=f"f{index}.txt",
+                size=1,
+                content_type="text/plain",
+                read=MagicMock(side_effect=AssertionError("too many files must not be read")),
+            )
+            for index in range(MAX_SESSION_UPLOAD_FILES + 1)
+        ]
+
+        with pytest.raises(ClientBlueException) as excinfo:
+            view.pv_files_upload(_request(method="POST", files=uploaded_files), pk="s1")
+
+        assert str(MAX_SESSION_UPLOAD_FILES) in excinfo.value.message
+        assert excinfo.value.STATUS_CODE == 400
+        for uploaded_file in uploaded_files:
+            uploaded_file.read.assert_not_called()
+        instance.upload_files.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # 异常映射：SandboxFileError 基类兜底为 ServerBlueException