sycommon-python-lib 0.2.7a43__tar.gz → 0.2.7a45__tar.gz

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a43
+Version: 0.2.7a45
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.7a43"
+version = "0.2.7a45"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/src/sycommon/agent/sandbox/file_ops.py

@@ -41,11 +41,10 @@ if TYPE_CHECKING:
 # 沙箱 API 路径前缀
 SANDBOX_API_PREFIX = "/v1/sandbox"
 
-# 下载/读取的文件大小预检上限（OOM 兜底，env 可调）。
-# adownload_files 整包读 + base64 解码，agent 调 download 工具读大文件会把全量拉进内存；
-# aread 有行数 limit 但无字节上限。下载/读取前先 astat 拿 size，超限返回 error 让 agent
-# 改用 head/grep/offset+limit 读。astat 异常 fail-open（warning 后继续，避免额外 RPC 失败阻断）。
-MAX_DOWNLOAD_BYTES = int(os.environ.get("SANDBOX_DOWNLOAD_MAX_BYTES", 100 * 1024 * 1024))
+# aread 读取的文件大小预检上限（OOM 兜底，env 可调）。
+# aread 走 /v1/sandbox/read，服务端另有 500KiB 截断（_READ_MAX_OUTPUT_BYTES），客户端 10MB
+# 是早死早超生的预检，防 agent 把大文件读进 LLM 上下文。astat 异常 fail-open。
+# 注：文件搬运（下载/分享/同步）已全部改走流式 adownload_to_file，无大小闸。
 MAX_READ_BYTES = int(os.environ.get("SANDBOX_READ_MAX_BYTES", 10 * 1024 * 1024))
 
 

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/src/sycommon/agent/sandbox/http_sandbox_backend.py

@@ -15,11 +15,48 @@ import sys
 import tempfile
 import threading
 import requests
+from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 from typing import Optional, List
 
 import aiohttp
 
+
+def _run_coro_sync(coro):
+    """在当前线程同步跑一个 coroutine（供同步方法调用异步实现）。
+
+    已在事件循环里则新起线程跑（避免嵌套 loop）；否则 new_event_loop 跑。
+    注意：coro 必须是「在任意线程新 loop 里都能跑」的——不要捕获外层 loop。
+    """
+    def _run_in_fresh_loop(c):
+        loop = asyncio.new_event_loop()
+        try:
+            return loop.run_until_complete(c)
+        finally:
+            loop.close()
+
+    try:
+        asyncio.get_running_loop()
+        in_loop = True
+    except RuntimeError:
+        in_loop = False
+
+    if in_loop:
+        import concurrent.futures
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
+            return ex.submit(_run_in_fresh_loop, coro).result()
+    return _run_in_fresh_loop(coro)
+
+
+@dataclass
+class FileDownloadResult:
+    """流式下载结果（adownload_to_file 返回）。不含 content bytes——文件已落盘到 dest。"""
+    path: str
+    dest: Optional[str] = None
+    size: int = 0
+    error: Optional[str] = None
+
+
 from sycommon.logging.kafka_log import SYLogger
 from sycommon.agent.sandbox.file_ops import FileOperationsMixin, SANDBOX_API_PREFIX
 
@@ -29,6 +66,9 @@ from deepagents.backends.protocol import (
     FileDownloadResponse,
     FileUploadResponse,
 )
+# FileDownloadResponse: 框架兼容（deepagents SkillsMiddleware/MemoryMiddleware 内部调
+# backend.adownload_files 批量下载 SKILL.md/memory，需此类型）。流式下载用 FileDownloadResult。
+
 
 # SSL 上下文
 try:
@@ -1174,72 +1214,107 @@ class HTTPSandboxBackend(FileOperationsMixin, SandboxBackendProtocol):
                         path=path, error=str(ex)))
             return results
 
-    def download_files(self, paths: List[str]) -> List[FileDownloadResponse]:
-        """下载多个文件"""
-        results = []
-        for path in paths:
+    # 流式下载块大小（与服务端 _STREAM_CHUNK 对齐）。客户端按此尺寸从 HTTP 响应流读块、
+    # 写盘，内存峰值 ≈ 单块，与文件总大小无关。替代旧整包 download_files（base64 全量进内存）。
+    _STREAM_CHUNK = int(os.environ.get("SANDBOX_STREAM_CHUNK", 1024 * 1024))
+
+    async def _download_stream_to_file(self, path: str, dest: str, *, timeout: int = None) -> "FileDownloadResult":
+        """流式下载单个沙箱文件到本地 dest（内部实现，含 failover）。
+
+        打 POST /v1/sandbox/download/stream，按 _STREAM_CHUNK 流式写盘，不进进程大内存。
+        失败时（瞬时网络抖动）原地重试一次，再切实例一次，策略对齐 _post_async_with_failover。
+        """
+        headers = self._build_headers()
+        http_timeout = (timeout or self._timeout) + 30
+        timeout_obj = aiohttp.ClientTimeout(total=http_timeout)
+
+        async def _do(base_url: str) -> "FileDownloadResult":
+            session = await self._get_async_session()
+            u = f"{base_url}{SANDBOX_API_PREFIX}/download/stream"
+            async with session.post(u, json={"path": path, "user_id": self._user_id},
+                                    headers=headers, timeout=timeout_obj) as resp:
+                if resp.status >= 400:
+                    text = await resp.text()
+                    return FileDownloadResult(
+                        path=path, error=f"HTTP {resp.status}: {text[:200]}")
+                with open(dest, "wb") as f:
+                    async for chunk in resp.content.iter_chunked(self._STREAM_CHUNK):
+                        f.write(chunk)
+            return FileDownloadResult(path=path, dest=dest,
+                                      size=os.path.getsize(dest))
+
+        last_exc: Optional[Exception] = None
+        for attempt in range(2):
             try:
-                result = self._post_sync(
-                    f"{SANDBOX_API_PREFIX}/download",
-                    {"path": path, "user_id": self._user_id}
-                )
-                if result.get("error"):
-                    results.append(FileDownloadResponse(
-                        path=path,
-                        error=result["error"]
-                    ))
-                else:
-                    content = base64.b64decode(result["content"])
-                    results.append(FileDownloadResponse(
-                        path=path, content=content))
-            except Exception as e:
-                results.append(FileDownloadResponse(path=path, error=str(e)))
-        return results
+                return await _do(self._base_url)
+            except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as e:
+                last_exc = e
+                SYLogger.warning(
+                    f"[Sandbox] 流式下载失败 (尝试 {attempt+1}/2): "
+                    f"{self._base_url} path={path} - {e}")
+                if attempt == 0:
+                    await asyncio.sleep(1)
+        switched = await asyncio.to_thread(self._refresh_from_nacos_and_switch_sync)
+        if switched:
+            try:
+                return await _do(self._base_url)
+            except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as e:
+                last_exc = e
+                SYLogger.warning(f"[Sandbox] 切实例后流式下载仍失败: {self._base_url} - {e}")
+        try:
+            os.unlink(dest)
+        except OSError:
+            pass
+        return FileDownloadResult(path=path, error=f"流式下载失败: {last_exc}")
 
-    async def adownload_files(self, paths: List[str], *, timeout: int = None) -> List[FileDownloadResponse]:
-        """异步下载多个文件（真正的异步实现）
+    async def adownload_to_file(self, path: str, dest: str, *, timeout: int = None) -> "FileDownloadResult":
+        """流式下载沙箱文件到本地 dest 路径（推荐用法，内存峰值≈单块）。
+
+        Args:
+            path: 沙箱内文件路径
+            dest: 本地目标文件路径（调用方负责清理；建议用 tempfile）
+            timeout: 超时秒数
 
-        🔑 OOM 兜底：每个文件下载前先 astat 拿 size（只查 size 不读内容），
-        超 MAX_DOWNLOAD_BYTES 直接返回 error，避免整包读 + base64 解码把全量拉进内存。
-        astat 异常 fail-open（warning 后继续下载）。
+        Returns:
+            FileDownloadResult(path, dest, size, error)。error 非 None 即失败。
         """
-        from sycommon.agent.sandbox.file_ops import MAX_DOWNLOAD_BYTES
-        results = []
-        for path in paths:
+        return await self._download_stream_to_file(path, dest, timeout=timeout)
+
+    # ---- 框架兼容：deepagents SkillsMiddleware/MemoryMiddleware 内部调
+    # backend.adownload_files(paths) 批量下载 SKILL.md/memory（第三方代码，无法改）。
+    # 这里保留 adownload_files/download_files 签名，内部走流式下载到临时文件再读 bytes，
+    # 内存安全。业务代码（get_share_url/minio_sync 等）应优先用 adownload_to_file 不读回。
+    async def adownload_files(self, paths: List[str], *, timeout: int = None) -> List[FileDownloadResponse]:
+        """批量下载（框架兼容）。内部流式落临时文件再读 bytes。"""
+        results: List[FileDownloadResponse] = []
+        for p in paths:
+            import tempfile as _tf
+            with _tf.NamedTemporaryFile(delete=False) as _tmp:
+                tmp_path = _tmp.name
             try:
-                # 大小预检：超限直接拒绝，不下载
-                try:
-                    st = await self.astat(path=path)
-                    if st and not getattr(st, "error", None) and getattr(st, "exists", False):
-                        fsize = getattr(st, "size", 0) or 0
-                        if fsize > MAX_DOWNLOAD_BYTES:
-                            cap_mb = MAX_DOWNLOAD_BYTES // (1024 * 1024)
-                            SYLogger.warning(f"[Sandbox] 文件过大拒绝下载: {path}, size={fsize}")
-                            results.append(FileDownloadResponse(
-                                path=path,
-                                error=f"file too large ({fsize // (1024*1024)}MB > {cap_mb}MB)"))
-                            continue
-                except Exception as se:
-                    SYLogger.warning(f"[Sandbox] astat 预检失败(继续下载): {se}")
-
-                result = await self._post_async_with_failover(
-                    f"{SANDBOX_API_PREFIX}/download",
-                    {"path": path, "user_id": self._user_id},
-                    timeout=timeout
-                )
-                if result.get("error"):
-                    results.append(FileDownloadResponse(
-                        path=path,
-                        error=result["error"]
-                    ))
+                r = await self.adownload_to_file(p, tmp_path, timeout=timeout)
+                if r.error:
+                    results.append(FileDownloadResponse(path=p, error=r.error))
                 else:
-                    content = base64.b64decode(result["content"])
-                    results.append(FileDownloadResponse(
-                        path=path, content=content))
+                    with open(tmp_path, "rb") as _f:
+                        results.append(FileDownloadResponse(path=p, content=_f.read()))
             except Exception as e:
-                results.append(FileDownloadResponse(path=path, error=str(e)))
+                results.append(FileDownloadResponse(path=p, error=str(e)))
+            finally:
+                try:
+                    os.unlink(tmp_path)
+                except OSError:
+                    pass
         return results
 
+    def download_files(self, paths: List[str]) -> List[FileDownloadResponse]:
+        """批量下载（同步包装，框架兼容）。"""
+        return _run_coro_sync(self.adownload_files(paths))
+
+    def download_to_file(self, path: str, dest: str, *, timeout: int = None) -> "FileDownloadResult":
+        """流式下载（同步包装）。在线程池独立 loop 里跑 adownload_to_file。"""
+        return _run_coro_sync(self.adownload_to_file(path, dest, timeout=timeout))
+
     # ============== 目录同步 ==============
 
     def _sync_sync(self) -> dict:
@@ -1552,18 +1627,26 @@ class HTTPSandboxBackend(FileOperationsMixin, SandboxBackendProtocol):
                 continue
             file_path = file_info["path"]
             try:
-                download_result = self._post_sync(
-                    f"{SANDBOX_API_PREFIX}/download",
-                    {"path": file_path, "user_id": self._user_id}
-                )
-                if download_result.get("error"):
-                    failed_count += 1
-                    errors.append(
-                        {"path": file_path, "error": download_result["error"]})
-                else:
-                    content = base64.b64decode(download_result["content"])
-                    files_to_sync.append((file_path, content))
-                    success_count += 1
+                # 流式下载到临时文件再读 bytes（旧实现走整包 /download base64 全量进内存）。
+                # 同步上下文：在线程池里跑 adownload_to_file。文件通常为技能/memory/历史小文件，
+                # 落盘读回可接受；大文件也不爆内存。
+                import tempfile as _tf
+                with _tf.NamedTemporaryFile(delete=False) as _tmp:
+                    tmp_path = _tmp.name
+                try:
+                    r = _run_coro_sync(self.adownload_to_file(file_path, tmp_path))
+                    if r.error:
+                        failed_count += 1
+                        errors.append({"path": file_path, "error": r.error})
+                    else:
+                        with open(tmp_path, "rb") as _f:
+                            files_to_sync.append((file_path, _f.read()))
+                        success_count += 1
+                finally:
+                    try:
+                        os.unlink(tmp_path)
+                    except OSError:
+                        pass
             except Exception as e:
                 failed_count += 1
                 errors.append({"path": file_path, "error": str(e)})

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/src/sycommon/agent/sandbox/minio_sync.py

@@ -196,6 +196,44 @@ class MinioSyncService(metaclass=SingletonMeta):
             SYLogger.error(f"[MinIO] Upload failed: {object_key}, error={e}")
             return False
 
+    def upload_stream(self, object_key: str, file_obj, length: int,
+                      content_type: str = "application/octet-stream") -> bool:
+        """从 file-like 对象流式分片上传（大文件专用，内存峰值≈5MB）。
+
+        MinIO put_object(length=size, part_size=0) 自动按 5MB 分片读 file_obj，
+        不一次性 read 全文件。已验证 50/100/200MB 上传内存峰值恒定 5MB。
+
+        Args:
+            object_key: MinIO 对象 key
+            file_obj: 有 read() 的二进制文件对象（如 open(path,'rb')）
+            length: 文件总字节数（必传，决定分片数）
+            content_type: MIME
+        """
+        if not self._client:
+            return False
+        try:
+            self._client.put_object(
+                bucket_name=self._bucket,
+                object_name=object_key,
+                data=file_obj,
+                length=length,
+                content_type=content_type,
+                # part_size=0 → minio 按 length 自动算 5MB 分片
+            )
+            SYLogger.info(
+                f"[MinIO] Stream upload success: {object_key}, size={length}")
+            return True
+        except Exception as e:
+            SYLogger.error(f"[MinIO] Stream upload failed: {object_key}, error={e}")
+            return False
+
+    async def aupload_stream(self, object_key: str, file_obj, length: int,
+                             content_type: str = "application/octet-stream") -> bool:
+        """异步流式分片上传"""
+        return await asyncio.to_thread(
+            self.upload_stream, object_key, file_obj, length, content_type)
+
+
     async def aupload_bytes(self, object_key: str, content: bytes,
                             content_type: str = "application/octet-stream") -> bool:
         """异步上传文件到 MinIO"""
@@ -606,30 +644,49 @@ class MinioSyncService(metaclass=SingletonMeta):
 
     async def _sync_one(self, sandbox_backend, fp: str,
                         existing_objects: dict, prefix: str) -> bool:
-        """同步单个文件：下载 + 上传，成功返回 True"""
+        """同步单个文件：流式下载到临时文件 + 流式上传 MinIO，成功返回 True。
+
+        内存峰值≈5MB（沙箱流式落盘 + MinIO 5MB 分片），与文件大小无关。
+        替代旧实现（adownload_files 整包读 + upload_bytes 全量进内存）。
+        """
+        import tempfile as _tf
+        tmp_path = None
         try:
-            results = await sandbox_backend.adownload_files([f"/{fp}"])
-            if not results or results[0].error or not results[0].content:
-                error = results[0].error if results else "unknown"
-                SYLogger.warning(f"[MinIO] 下载沙箱文件失败: {fp}, error={error}")
+            # MinIO 已有且 size 相同时，用 astat 的 size 先判跳过（省一次下载）
+            st = await sandbox_backend.astat(path=f"/{fp}")
+            if not st or getattr(st, "error", None) or not getattr(st, "exists", False):
+                SYLogger.warning(f"[MinIO] astat 沙箱文件失败/不存在: {fp}")
                 return False
-
-            content = results[0].content
-
-            # 如果 MinIO 已有且 size 相同，跳过
-            if fp in existing_objects and existing_objects[fp].size == len(content):
+            fsize = getattr(st, "size", 0) or 0
+            if fp in existing_objects and existing_objects[fp].size == fsize:
                 return False
 
             mime_type, _ = mimetypes.guess_type(fp)
             object_key = f"{prefix}{fp}"
 
-            return await self.aupload_bytes(
-                object_key, content,
-                content_type=mime_type or "application/octet-stream",
-            )
+            # 流式下载到临时文件
+            with _tf.NamedTemporaryFile(delete=False) as tmp:
+                tmp_path = tmp.name
+            r = await sandbox_backend.adownload_to_file(f"/{fp}", tmp_path)
+            if r.error or r.size == 0:
+                SYLogger.warning(f"[MinIO] 流式下载沙箱文件失败: {fp}, error={r.error}")
+                return False
+            # 流式分片上传 MinIO（open 后由 minio 内部 5MB 分片读，不进大内存）
+            with open(tmp_path, "rb") as f:
+                return await self.aupload_stream(
+                    object_key, f, r.size,
+                    content_type=mime_type or "application/octet-stream",
+                )
         except Exception as e:
             SYLogger.warning(f"[MinIO] 同步文件异常: {fp}, error={e}")
             return False
+        finally:
+            if tmp_path:
+                try:
+                    os.unlink(tmp_path)
+                except OSError:
+                    pass
+
 
     # ============== 用户上传文件直推（fire-and-forget 后台）==============
     # 用户刚传上来的文件优先落 MinIO：不等会话结束扫描，免得沙箱中途崩溃丢文件。

{sycommon_python_lib-0.2.7a43 → sycommon_python_lib-0.2.7a45}/src/sycommon/middleware/sandbox.py

@@ -33,14 +33,14 @@ import shlex
 import threading
 from datetime import datetime
 from fastapi import FastAPI, APIRouter, Request, Header
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, JSONResponse
 from typing import Dict
 
 from sycommon.logging.kafka_log import SYLogger
 from sycommon.models.sandbox import (
     ExecuteRequest, ExecuteResponse,
     UploadRequest, UploadResponse,
-    DownloadRequest, DownloadResponse,
+    DownloadRequest,
     DeleteRequest, DeleteResponse,
     FileInfo, LsRequest, LsResponse,
     TreeNode, TreeRequest, TreeResponse,
@@ -1238,30 +1238,60 @@ def setup_sandbox_handler(app: FastAPI, config: dict = None):
 
         return await asyncio.to_thread(_batch)
 
-    @sandbox_router.post("/download", response_model=DownloadResponse)
-    async def download_file(req: DownloadRequest):
-        """下载文件"""
+    # 流式下载块大小。服务端按此大小读文件块吐给客户端，客户端同尺寸写盘，
+    # 双方内存峰值恒定 ≈ 块大小，与文件总大小无关（GB 级文件也不爆）。
+    # 替代旧整包 /download（os.ReadFile + base64 全量进内存）。
+    _STREAM_CHUNK = int(os.environ.get("SANDBOX_STREAM_CHUNK", 1024 * 1024))
+
+    @sandbox_router.post("/download/stream")
+    async def download_stream(req: DownloadRequest):
+        """流式下载文件（替代旧整包 /download）。
+
+        返回 StreamingResponse，按 _STREAM_CHUNK 块吐原始字节（不 base64）。
+        内存峰值 ≈ 单块，支持任意大小文件。错误用 HTTP 状态码 + JSON body 表达：
+          400 路径非法 / 404 文件不存在 / 500 服务端异常。
+        鉴权与其它接口一致（客户端带 X-Sandbox-Token，本服务端内网信任不校验）。
+        """
         workspace = await aget_user_workspace(req.user_id)
         try:
             file_path = resolve_sandbox_path(req.path, workspace)
         except ValueError as e:
-            return DownloadResponse(path=req.path, error=str(e))
+            SYLogger.warning(f"[Sandbox Server] download/stream 路径非法: {e}")
+            return JSONResponse(status_code=400, content={"error": str(e)})
 
-        def _download():
+        # 预检：文件存在性 + 拿 size（流式吐需要 Content-Length）
+        def _check():
             if not os.path.isfile(file_path):
-                return DownloadResponse(path=file_path, error="file_not_found")
-            with open(file_path, 'rb') as f:
-                content = f.read()
-            return DownloadResponse(
-                path=file_path,
-                content=base64.b64encode(content).decode(),
-                error=None
-            )
-
+                return None, "file_not_found"
+            return os.path.getsize(file_path), None
         try:
-            return await asyncio.to_thread(_download)
+            size, err = await asyncio.to_thread(_check)
         except Exception as e:
-            return DownloadResponse(path=file_path, error=str(e))
+            SYLogger.error(f"[Sandbox Server] download/stream stat 异常: {e}")
+            return JSONResponse(status_code=500, content={"error": str(e)})
+        if err:
+            return JSONResponse(status_code=404, content={"error": err})
+
+        SYLogger.info(
+            f"[Sandbox Server] download/stream: {file_path} (user={req.user_id}, size={size})")
+
+        async def _gen():
+            # 文件 IO 在线程池跑，避免阻塞事件循环。一次只持有一块在内存。
+            loop = asyncio.get_event_loop()
+            f = await loop.run_in_executor(None, open, file_path, "rb")
+            try:
+                while True:
+                    chunk = await loop.run_in_executor(None, f.read, _STREAM_CHUNK)
+                    if not chunk:
+                        break
+                    yield chunk
+            finally:
+                await loop.run_in_executor(None, f.close)
+
+        headers = {"Content-Length": str(size)}
+        return StreamingResponse(
+            _gen(), media_type="application/octet-stream", headers=headers)
+
 
     @sandbox_router.post("/tree", response_model=TreeResponse)
     async def tree_dir(req: TreeRequest):