sycommon-python-lib 0.2.7a53__tar.gz → 0.2.7a54__tar.gz

{sycommon_python_lib-0.2.7a53 → sycommon_python_lib-0.2.7a54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a53
+Version: 0.2.7a54
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.7a53 → sycommon_python_lib-0.2.7a54}/pyproject.toml

@@ -1,67 +1,67 @@
-[project]
-name = "sycommon-python-lib"
-version = "0.2.7a53"
-description = "Add your description here"
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "aio-pika==10.0.1",
-    "aiohttp==3.14.3",
-    "aiomysql==0.3.2",
-    "anyio==4.14.2",
-    "decorator==5.3.1",
-    "deepagents==0.7.1",
-    "elasticsearch==9.4.1",
-    "fastapi==0.141.1",
-    "jinja2==3.1.6",
-    "kafka-python==3.0.9",
-    "langchain==1.3.14",
-    "langchain-core==1.5.3",
-    "langchain-openai==1.4.1",
-    "langfuse==4.14.2",
-    "langgraph==1.2.10",
-    "langgraph-checkpoint-postgres==3.1.1",
-    "langgraph-checkpoint-redis==0.5.1",
-    "ldap3==2.9.1",
-    "loguru==0.7.3",
-    "mysql-connector-python==26.7.0",
-    "nacos-sdk-python==2.0.11,<3.0",
-    "psutil==7.2.2",
-    "pyxxl==0.4.8",
-    "pydantic==2.13.4",
-    "python-dotenv==1.2.2",
-    "python-multipart==0.0.32",
-    "pyyaml==6.0.3",
-    "redis==7.4.0",
-    "sentry-sdk[fastapi]==2.66.1",
-    "sqlalchemy[asyncio]==2.0.51",
-    "starlette[full]==1.3.1",
-    "tiktoken==0.13.0",
-    "uvicorn==0.52.1",
-    "wecom-aibot-python-sdk==1.0.2",
-    "twine==7.0.0",
-    "minio==7.2.20",
-    "langchain-mcp-adapters==0.3.1",
-    "fastapi-mcp==0.4.0",
-    "psycopg[binary,pool]==3.3.4",
-    "agent-client-protocol==0.12.0",
-    "pypinyin==0.55.0",
-]
-
-[tool.setuptools]
-packages = {find = {where = ["src"], exclude = ["*.__pycache__", "*.__pycache__.*"]}}
-package-data = {command = ["templates/**/*", "templates/**/.*"], sycommon = ["llm/tiktoken_cache/*"]}
-
-[build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
-
-[dependency-groups]
-dev = [
-    "pytest>=9.0.3",
-]
-
-[project.scripts]
-sycommon = "command.cli:main"
-sycli = "sycli.cli:main"
-sypublish = "nexus:main"
+[project]
+name = "sycommon-python-lib"
+version = "0.2.7a54"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "aio-pika==10.0.1",
+    "aiohttp==3.14.3",
+    "aiomysql==0.3.2",
+    "anyio==4.14.2",
+    "decorator==5.3.1",
+    "deepagents==0.7.1",
+    "elasticsearch==9.4.1",
+    "fastapi==0.141.1",
+    "jinja2==3.1.6",
+    "kafka-python==3.0.9",
+    "langchain==1.3.14",
+    "langchain-core==1.5.3",
+    "langchain-openai==1.4.1",
+    "langfuse==4.14.2",
+    "langgraph==1.2.10",
+    "langgraph-checkpoint-postgres==3.1.1",
+    "langgraph-checkpoint-redis==0.5.1",
+    "ldap3==2.9.1",
+    "loguru==0.7.3",
+    "mysql-connector-python==26.7.0",
+    "nacos-sdk-python==2.0.11,<3.0",
+    "psutil==7.2.2",
+    "pyxxl==0.4.8",
+    "pydantic==2.13.4",
+    "python-dotenv==1.2.2",
+    "python-multipart==0.0.32",
+    "pyyaml==6.0.3",
+    "redis==7.4.0",
+    "sentry-sdk[fastapi]==2.66.1",
+    "sqlalchemy[asyncio]==2.0.51",
+    "starlette[full]==1.3.1",
+    "tiktoken==0.13.0",
+    "uvicorn==0.52.1",
+    "wecom-aibot-python-sdk==1.0.2",
+    "twine==7.0.0",
+    "minio==7.2.20",
+    "langchain-mcp-adapters==0.3.1",
+    "fastapi-mcp==0.4.0",
+    "psycopg[binary,pool]==3.3.4",
+    "agent-client-protocol==0.12.0",
+    "pypinyin==0.55.0",
+]
+
+[tool.setuptools]
+packages = {find = {where = ["src"], exclude = ["*.__pycache__", "*.__pycache__.*"]}}
+package-data = {command = ["templates/**/*", "templates/**/.*"], sycommon = ["llm/tiktoken_cache/*"]}
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3",
+]
+
+[project.scripts]
+sycommon = "command.cli:main"
+sycli = "sycli.cli:main"
+sypublish = "nexus:main"

{sycommon_python_lib-0.2.7a53 → sycommon_python_lib-0.2.7a54}/src/sycommon/middleware/sandbox.py

@@ -108,6 +108,45 @@ _PIP_TMP_MIN_AGE_SECONDS = 10 * 60
 # 不再用周期 60s 轮询——启动 + 对话结束两个事件已覆盖所有产生时机。
 
 
+# ============== 流式上传并发限流 ==============
+# 两层信号量：全局封顶防多用户同时上传打爆服务端线程池/IO；per-user 限流防
+# 单用户独占全局额度。超出额度的请求在信号量上排队（asyncio.Semaphore 是 FIFO
+# 公平等待，不会饿死），不是直接拒绝。
+#
+# 配置走环境变量（默认值：全局 50 / 单用户 3）：
+#   SANDBOX_UPLOAD_GLOBAL_MAX  — 全局并发上传上限
+#   SANDBOX_UPLOAD_PER_USER    — 单用户并发上传上限
+#
+# 🔑 多 worker 注意：每个 worker 进程各自独立计数（进程级信号量，非跨进程共享）。
+# 实际全局上限 ≈ workers × SANDBOX_UPLOAD_GLOBAL_MAX。沙箱按 user_id 哈希到固定
+# worker（load_balance），故单用户的真实上限仍是 SANDBOX_UPLOAD_PER_USER，不受多 worker 影响。
+_UPLOAD_GLOBAL_MAX = int(os.environ.get("SANDBOX_UPLOAD_GLOBAL_MAX", 50))
+_UPLOAD_PER_USER = int(os.environ.get("SANDBOX_UPLOAD_PER_USER", 3))
+
+# 全局信号量：惰性创建（绑定首次调用时的 event loop；单 loop 部署下不变）。
+# 模块顶层直接 asyncio.Semaphore() 在某些导入时机（无 running loop）会报错，故延迟。
+_upload_global_sem: "asyncio.Semaphore | None" = None
+# per-user 信号量缓存：user_id -> Semaphore。不同用户独立计数、互不抢占。
+_upload_user_sems: "Dict[str, asyncio.Semaphore]" = {}
+
+
+def _get_upload_global_sem() -> "asyncio.Semaphore":
+    """惰性创建全局上传信号量。"""
+    global _upload_global_sem
+    if _upload_global_sem is None:
+        _upload_global_sem = asyncio.Semaphore(_UPLOAD_GLOBAL_MAX)
+    return _upload_global_sem
+
+
+def _get_upload_user_sem(user_id: str) -> "asyncio.Semaphore":
+    """惰性创建/取回某用户的上传信号量。不存在则新建（不会回收，user_id 域有限，可忽略）。"""
+    sem = _upload_user_sems.get(user_id)
+    if sem is None:
+        sem = asyncio.Semaphore(_UPLOAD_PER_USER)
+        _upload_user_sems[user_id] = sem
+    return sem
+
+
 async def _cleanup_finished_processes():
     """周期清理已完成的过期后台进程记录（仅进程记录，不含 pip tmp）。
 
@@ -1394,75 +1433,95 @@ def setup_sandbox_handler(app: FastAPI, config: dict = None):
         支持任意大小文件。元数据（path/user_id/size）走 query 参数，不进 body——
         避免框架/中间件预缓冲 JSON body 重新引入 OOM（对齐 /download/stream 的反向思路）。
         错误用 HTTP 状态码 + JSON body 表达：400 路径非法 / 500 写入或长度不符。
+
+        并发限流：两层 asyncio.Semaphore（全局 _UPLOAD_GLOBAL_MAX / 单用户
+        _UPLOAD_PER_USER）。超出额度的请求在此排队等待，不拒绝。先抢全局额度、
+        再抢本用户额度，释放顺序相反，避免全局额度被等待用户额度的请求空占。
         """
-        workspace = await aget_user_workspace(user_id)
+        # 并发限流在路径校验之前：让超额请求尽早排队，而非先做路径解析再排队。
+        # （路径非法的请求量极小，不值得为此打乱限流顺序。）
+        global_sem = _get_upload_global_sem()
+        user_sem = _get_upload_user_sem(user_id)
+        await global_sem.acquire()
         try:
-            file_path = resolve_sandbox_path(path, workspace)
-        except ValueError as e:
-            SYLogger.warning(f"[Sandbox Server] upload/stream 路径非法: {e}")
-            return JSONResponse(status_code=400, content={"error": str(e)})
+            await user_sem.acquire()
+        except BaseException:
+            # acquire user_sem 失败（如被取消）必须先放全局额度，避免泄漏。
+            global_sem.release()
+            raise
+        try:
+            workspace = await aget_user_workspace(user_id)
+            try:
+                file_path = resolve_sandbox_path(path, workspace)
+            except ValueError as e:
+                SYLogger.warning(f"[Sandbox Server] upload/stream 路径非法: {e}")
+                return JSONResponse(status_code=400, content={"error": str(e)})
 
-        # 预建父目录（磁盘 IO 不阻塞事件循环）
-        parent = os.path.dirname(file_path)
+            # 预建父目录（磁盘 IO 不阻塞事件循环）
+            parent = os.path.dirname(file_path)
 
-        def _prepare():
-            if parent:
-                os.makedirs(parent, exist_ok=True)
-        try:
-            await asyncio.to_thread(_prepare)
-        except OSError as e:
-            SYLogger.error(f"[Sandbox Server] upload/stream 建目录失败: {e}")
-            return JSONResponse(status_code=500, content={"error": str(e)})
+            def _prepare():
+                if parent:
+                    os.makedirs(parent, exist_ok=True)
+            try:
+                await asyncio.to_thread(_prepare)
+            except OSError as e:
+                SYLogger.error(f"[Sandbox Server] upload/stream 建目录失败: {e}")
+                return JSONResponse(status_code=500, content={"error": str(e)})
 
-        loop = asyncio.get_event_loop()
-        # O_NOFOLLOW 防 symlink 写劫持（对齐 /upload、/batch_upload）。
-        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
+            loop = asyncio.get_event_loop()
+            # O_NOFOLLOW 防 symlink 写劫持（对齐 /upload、/batch_upload）。
+            flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
 
-        f = None
-        written = 0
-        try:
-            # os.fdopen(fd, 'wb')：二进制模式（Windows 不做 \n→\r\n 翻译），
-            # 内部按缓冲写盘并消化 partial write，比裸 os.write 更稳（对齐 /upload）。
-            fd = await loop.run_in_executor(None, os.open, file_path, flags, 0o644)
-            f = await loop.run_in_executor(None, os.fdopen, fd, 'wb')
-            async for chunk in request.stream():
-                if not chunk:
-                    continue
-                await loop.run_in_executor(None, f.write, chunk)
-                written += len(chunk)
-            await loop.run_in_executor(None, f.close)
             f = None
-        except Exception as e:
-            # 客户端中途断连 / 写盘失败：关句柄 + 清理半成品文件（对齐 download_stream 的 finally 清理）
-            SYLogger.error(
-                f"[Sandbox Server] upload/stream 写入异常: {file_path} - {type(e).__name__}: {e}")
-            if f is not None:
+            written = 0
+            try:
+                # os.fdopen(fd, 'wb')：二进制模式（Windows 不做 \n→\r\n 翻译），
+                # 内部按缓冲写盘并消化 partial write，比裸 os.write 更稳（对齐 /upload）。
+                fd = await loop.run_in_executor(None, os.open, file_path, flags, 0o644)
+                f = await loop.run_in_executor(None, os.fdopen, fd, 'wb')
+                async for chunk in request.stream():
+                    if not chunk:
+                        continue
+                    await loop.run_in_executor(None, f.write, chunk)
+                    written += len(chunk)
+                await loop.run_in_executor(None, f.close)
+                f = None
+            except Exception as e:
+                # 客户端中途断连 / 写盘失败：关句柄 + 清理半成品文件（对齐 download_stream 的 finally 清理）
+                SYLogger.error(
+                    f"[Sandbox Server] upload/stream 写入异常: {file_path} - {type(e).__name__}: {e}")
+                if f is not None:
+                    try:
+                        await loop.run_in_executor(None, f.close)
+                    except OSError:
+                        pass
                 try:
-                    await loop.run_in_executor(None, f.close)
+                    await loop.run_in_executor(None, os.unlink, file_path)
                 except OSError:
                     pass
-            try:
-                await loop.run_in_executor(None, os.unlink, file_path)
-            except OSError:
-                pass
-            return JSONResponse(status_code=500, content={"error": str(e)})
+                return JSONResponse(status_code=500, content={"error": str(e)})
 
-        # size 给定时校验长度（防客户端截断后留下残缺文件）
-        if size is not None and written != size:
-            SYLogger.warning(
-                f"[Sandbox Server] upload/stream 长度不符: {file_path} "
-                f"written={written} expected={size}，清理半成品")
-            try:
-                await loop.run_in_executor(None, os.unlink, file_path)
-            except OSError:
-                pass
-            return JSONResponse(
-                status_code=500,
-                content={"error": f"size mismatch: written={written} expected={size}"})
+            # size 给定时校验长度（防客户端截断后留下残缺文件）
+            if size is not None and written != size:
+                SYLogger.warning(
+                    f"[Sandbox Server] upload/stream 长度不符: {file_path} "
+                    f"written={written} expected={size}，清理半成品")
+                try:
+                    await loop.run_in_executor(None, os.unlink, file_path)
+                except OSError:
+                    pass
+                return JSONResponse(
+                    status_code=500,
+                    content={"error": f"size mismatch: written={written} expected={size}"})
 
-        SYLogger.info(
-            f"[Sandbox Server] upload/stream 完成: {file_path} (user={user_id}, size={written})")
-        return {"path": path, "size": written}
+            SYLogger.info(
+                f"[Sandbox Server] upload/stream 完成: {file_path} (user={user_id}, size={written})")
+            return {"path": path, "size": written}
+        finally:
+            # 释放顺序：先放用户额度，再放全局额度（与获取相反，避免全局额度空占）
+            user_sem.release()
+            global_sem.release()
 
     @sandbox_router.post("/tree", response_model=TreeResponse)
     async def tree_dir(req: TreeRequest):

{sycommon_python_lib-0.2.7a53 → sycommon_python_lib-0.2.7a54}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a53
+Version: 0.2.7a54
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown