sycommon-python-lib 0.2.10a4__tar.gz → 0.2.10b6__tar.gz

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.10a4
+Version: 0.2.10b6
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.10a4"
+version = "0.2.10b6"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/src/sycommon/agent/sandbox/minio_sync.py

@@ -886,6 +886,26 @@ class MinioSyncService(metaclass=SingletonMeta):
 
     # ============== 反向同步（MinIO → 沙箱） ==============
 
+    def _download_object_to_file(self, key: str, dest: str) -> int:
+        """流式下载 MinIO 对象到本地文件（同步，跑在 to_thread），返回字节数。
+
+        替代旧 resp.read() 整文件进内存：大附件场景一批 5 路并发 × 全量驻留
+        = 数百 MB~GB 级 RSS 峰值（打爆内存的根因）。流式写盘后内存峰值 ≈ 单块，
+        与文件大小无关；与 _sync_one 的 adownload_to_file 落盘模式对称。
+        块大小 1MB（5 路并发 × 1MB = 常驻 ~5MB 级）。
+        """
+        resp = self._client.get_object(self._bucket, key)
+        try:
+            total = 0
+            with open(dest, "wb") as f:
+                for chunk in resp.stream(1 * 1024 * 1024):
+                    f.write(chunk)
+                    total += len(chunk)
+            return total
+        finally:
+            resp.close()
+            resp.release_conn()
+
     async def restore_from_minio(self, sandbox_backend, user_id: str) -> int:
         """将 MinIO 中的文件恢复到沙箱
 
@@ -953,9 +973,14 @@ class MinioSyncService(metaclass=SingletonMeta):
         #     打满线程池、挤占主请求路径的沙箱操作。
         # 下载协程先抢本用户 5 槽、再抢全局 100 槽，故实际并发 = min(本用户待下载, 全局剩余)。
         #
-        # 🔑 OOM 兜底：旧实现 gather 所有文件内容进 downloaded 列表后才上传，用户文件多/大
-        #   一次几 GB 进内存。改为【分批流水线】：下 _RESTORE_PER_USER 个→立即上传→字节出作用域
-        #   →下一批。同时按 list_objects 的 size 累加 total_bytes_seen，超 _MAX_RESTORE_BYTES 跳过剩余。
+        # 🔑 OOM 治理（两处放大器都已拔掉）：
+        #   - 上传侧旧实现 gather 所有文件内容进 downloaded 列表后才 base64 整包传沙箱，
+        #     用户文件多/大时一次几 GB 进内存 → 改【分批流水线】+ 每批立即上传；
+        #   - 下载侧旧实现 resp.read() 把整文件读进内存，单个大附件即百 MB 级驻留
+        #     → 改【流式落盘】：MinIO 5MB 块写临时文件 → aupload_stream_from_disk
+        #     逐块读盘推沙箱 → 删临时文件，内存峰值 ≈ 单块，与文件大小无关。
+        #   另按 list_objects 的 size 累加 total_bytes_seen，超 _MAX_RESTORE_BYTES 跳过剩余
+        #   （仅 warning，会话末尾全量扫描兜底）。
         restored = 0
         global_sem = _get_restore_download_sem()
         per_user_sem = asyncio.Semaphore(_RESTORE_PER_USER)
@@ -964,26 +989,43 @@ class MinioSyncService(metaclass=SingletonMeta):
         obj_size_map = {o.object_name[len(prefix):]: (o.size or 0) for o in objects}
 
         async def _download_one(rel_path: str):
-            """下载单个 MinIO 对象到内存，返回 (沙箱绝对路径, 内容) 或 None。"""
+            """下载单个 MinIO 对象【流式落盘】再流式上传沙箱，成功 True 失败 None。
+
+            全程不把整文件 bytes 攒进内存：MinIO 5MB 块写临时文件 →
+            aupload_stream_from_disk 逐块读盘推沙箱 → 删临时文件。
+            """
+            tmp_path = None
             async with per_user_sem, global_sem:
                 try:
-                    def _download_object(bucket, key):
-                        resp = self._client.get_object(bucket, key)
-                        try:
-                            return resp.read()
-                        finally:
-                            resp.close()
-                            resp.release_conn()
-
-                    content = await asyncio.to_thread(
-                        _download_object, self._bucket, f"{prefix}{rel_path}"
+                    import tempfile as _tf
+                    with _tf.NamedTemporaryFile(delete=False) as tmp:
+                        tmp_path = tmp.name
+                    fsize = await asyncio.to_thread(
+                        self._download_object_to_file,
+                        f"{prefix}{rel_path}", tmp_path
+                    )
+                    if fsize == 0:
+                        SYLogger.warning(f"[MinIO] 下载文件为空/失败: {rel_path}")
+                        return None
+                    r = await sandbox_backend.aupload_stream_from_disk(
+                        f"/{rel_path}", tmp_path
                     )
-                    return (f"/{rel_path}", content)
+                    if getattr(r, "error", None):
+                        SYLogger.warning(
+                            f"[MinIO] 上传到沙箱失败: {rel_path}, error={r.error}")
+                        return None
+                    return rel_path
                 except Exception as e:
                     SYLogger.warning(f"[MinIO] 下载文件失败: {rel_path}, error: {e}")
                     return None
+                finally:
+                    if tmp_path:
+                        try:
+                            os.unlink(tmp_path)
+                        except OSError:
+                            pass
 
-        # 分批流水线：下 N 个 → 传 N 个 → 字节随 downloaded/download_results 出作用域释放
+        # 分批流水线：下 N 个 → 传 N 个 → 临时文件随 _download_one finally 删除
         total_bytes_seen = 0
         capped = False
         batch_size = _RESTORE_PER_USER
@@ -1010,18 +1052,7 @@ class MinioSyncService(metaclass=SingletonMeta):
                 *[_download_one(rp) for rp in batch_paths]
             )
             downloaded = [r for r in download_results if r is not None]
-            if not downloaded:
-                continue
-            try:
-                results = await sandbox_backend.aupload_files(downloaded)
-                for r in results:
-                    if not r.error:
-                        restored += 1
-                    else:
-                        SYLogger.warning(
-                            f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
-            except Exception as e:
-                SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
+            restored += len(downloaded)
             # downloaded / download_results 到此出作用域，下一批前可被 GC
 
         if capped:

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/src/sycommon/auth/ldap_service.py

@@ -70,6 +70,10 @@ class LDAPAuthResult(BaseModel):
     error: Optional[str] = None
 
 
+class LDAPAuthError(Exception):
+    """单台 LDAP 服务器认证失败(连接或绑定),消息交由上层分类。"""
+
+
 class LDAPAuthService:
     """LDAP 认证服务
 
@@ -89,6 +93,9 @@ class LDAPAuthService:
     def __init__(self, config: LDAPConfig):
         self.config = config
         self._connection = None
+        # 记住最近一次成功过的服务器地址:认证成功/搜索成功后直连它,
+        # 失败才重新竞速——避免每一步都为死机器付出 connect 超时的代价。
+        self._healthy_url: Optional[str] = None
 
     async def authenticate(
         self,
@@ -97,6 +104,9 @@ class LDAPAuthService:
     ) -> LDAPAuthResult:
         """验证用户凭证
 
+        所有 LDAP 服务器并发 connect+bind,谁先成功立即返回;
+        只有在任何一台都不成功时,才等全部结果做错误分类。
+
         Args:
             username: 用户名（域账号）
             password: 密码
@@ -105,85 +115,112 @@ class LDAPAuthService:
             LDAPAuthResult: 认证结果
         """
         try:
-            # 在线程池中执行同步 LDAP 操作
-            result = await asyncio.get_event_loop().run_in_executor(
-                None,
-                self._authenticate_sync,
-                username,
-                password,
-            )
+            # 在线程池中并发执行各服务器的同步 LDAP 操作
+            loop = asyncio.get_event_loop()
+            result = await self._authenticate_race(loop, username, password)
             return result
         except Exception as e:
             SYLogger.error(f"[LDAP] 认证异常: {e}")
             return LDAPAuthResult(success=False, error=str(e))
 
-    def _authenticate_sync(self, username: str, password: str) -> LDAPAuthResult:
-        """同步执行 LDAP 认证"""
-        import ldap3
-        from ldap3.core.exceptions import LDAPException
-
-        # 构建用户 DN
-        user_dn = self._build_user_dn(username)
+    async def _authenticate_race(
+        self, loop: asyncio.AbstractEventLoop, username: str, password: str
+    ) -> LDAPAuthResult:
+        """并发对每台服务器执行 connect+bind(+成功后取用户信息)。
 
-        # 尝试连接各个 LDAP 服务器
-        # 优先返回认证类错误（invalidCredentials 等），而非连接类错误（timeout）。
-        # 避免备用服务器连接超时覆盖了主服务器的真实认证错误（密码错误），误导用户。
-        errors = []
-        for url in self.config.urls:
+        快路径:任一台成功立即返回(LDAPAuthResult.success=True);
+        活机器报认证类错误(密码错误)也立即返回——密码在任何一台 DC
+        上都错,等其它服务器不会改变结论,不陪死机器耗超时;
+        仅全部失败(连接类错误居多)时收齐错误做分类。
+        """
+        auth_keywords = [
+            "invalidcredentials",
+            "invalidattribute",
+            "insufficientaccess",
+            "密码",
+            "credential",
+        ]
+
+        def _is_auth_error(msg: str) -> bool:
+            lm = msg.lower()
+            return any(kw in lm for kw in auth_keywords)
+
+        def _try_one(url: str):
+            """单台服务器认证;成功返回 (url, result, None),失败返回 (url, None, err)。"""
             try:
-                # 创建连接（移除末尾的斜杠，设置连接超时）
-                clean_url = url.rstrip("/")
-                server = ldap3.Server(
-                    clean_url,
-                    get_info=ldap3.ALL,
-                    connect_timeout=5,  # 连接超时 5 秒
-                )
-                conn = ldap3.Connection(
-                    server,
-                    user=user_dn,
-                    password=password,
-                    auto_bind=True,
-                    receive_timeout=10,  # 接收超时 10 秒
-                )
-
-                # 认证成功，获取用户信息
-                user_info = self._get_user_info(conn, username)
-                conn.unbind()
-
-                SYLogger.info(f"[LDAP] 用户 {username} 认证成功")
-                return LDAPAuthResult(
-                    success=True,
-                    user=user_info,
-                )
-
-            except LDAPException as e:
-                errors.append((url, str(e)))
-                SYLogger.warning(f"[LDAP] 连接 {url} 失败: {e}")
-                continue
+                result = self._authenticate_one(username, password, url)
+                return url, result, None
+            except LDAPAuthError as e:
+                return url, None, str(e)
+
+        tasks = [
+            loop.run_in_executor(None, _try_one, url.rstrip("/"))
+            for url in self.config.urls
+        ]
+
+        errors: list = []
+        for fut in asyncio.as_completed(tasks):
+            url, result, err = await fut
+            if err is None:
+                self._healthy_url = url
+                return result
+            errors.append((url, err))
+            # 认证类错误是终局结论,立即返回,不等仍在 dial 的死机器
+            if _is_auth_error(err):
+                SYLogger.error(f"[LDAP] 用户 {username} 认证失败: {err}")
+                return LDAPAuthResult(success=False, error=f"认证失败: {err}")
 
-        # 优先返回认证类错误（密码错误等），而不是连接超时
-        # 按优先级排序：invalidCredentials / invalidAttribute / insufficientAccess 优先
-        auth_keywords = ["invalidcredentials", "invalidattribute", "insufficientaccess", "密码", "credential"]
+        # 全部失败:优先返回认证类错误,其次第一台(通常是主服务器)的连接错误
         for url, err in errors:
-            if any(kw in err.lower() for kw in auth_keywords):
+            if _is_auth_error(err):
                 SYLogger.error(f"[LDAP] 用户 {username} 认证失败: {err}")
-                return LDAPAuthResult(
-                    success=False,
-                    error=f"认证失败: {err}",
-                )
-
-        # 全是连接类错误，返回第一个（通常是主服务器）
+                return LDAPAuthResult(success=False, error=f"认证失败: {err}")
         if errors:
             first_err = errors[0][1]
             SYLogger.error(f"[LDAP] 用户 {username} 认证失败: {first_err}")
-            return LDAPAuthResult(
-                success=False,
-                error=f"认证失败: {first_err}",
-            )
+            return LDAPAuthResult(success=False, error=f"认证失败: {first_err}")
 
         SYLogger.error(f"[LDAP] 用户 {username} 认证失败: 未知错误")
         return LDAPAuthResult(success=False, error="认证失败: 未知错误")
 
+    def _authenticate_one(
+        self, username: str, password: str, url: str
+    ) -> LDAPAuthResult:
+        """对单台 LDAP 服务器执行 connect+bind+取用户信息。
+
+        Raises:
+            LDAPAuthError: 连接或绑定失败(含密码错误),错误消息交由上层分类
+        """
+        import ldap3
+        from ldap3.core.exceptions import LDAPException
+
+        user_dn = self._build_user_dn(username)
+
+        try:
+            server = ldap3.Server(
+                url,
+                get_info=ldap3.ALL,
+                connect_timeout=5,  # 连接超时 5 秒
+            )
+            conn = ldap3.Connection(
+                server,
+                user=user_dn,
+                password=password,
+                auto_bind=True,
+                receive_timeout=10,  # 接收超时 10 秒
+            )
+
+            # 认证成功,获取用户信息(同一连接内完成,不再二次拨号)
+            user_info = self._get_user_info(conn, username)
+            conn.unbind()
+
+            SYLogger.info(f"[LDAP] 用户 {username} 经 {url} 认证成功")
+            return LDAPAuthResult(success=True, user=user_info)
+
+        except LDAPException as e:
+            SYLogger.warning(f"[LDAP] 连接 {url} 失败: {e}")
+            raise LDAPAuthError(str(e)) from e
+
     def _build_user_dn(self, username: str) -> str:
         """构建用户 DN
 
@@ -279,16 +316,21 @@ class LDAPAuthService:
             return None
 
     def _search_user_sync(self, username: str) -> Optional[LDAPUser]:
-        """同步执行用户搜索"""
+        """同步执行用户搜索;记住的 healthy 服务器优先,失败再逐台回退"""
         import ldap3
         from ldap3.core.exceptions import LDAPException
 
         bind_user = self._build_bind_dn(self.config.username)
 
-        for url in self.config.urls:
+        # healthy 机器优先(它最近验证过可用),其余按配置顺序回退
+        urls = [u.rstrip("/") for u in self.config.urls]
+        if self._healthy_url and self._healthy_url in urls:
+            urls = [self._healthy_url] + [u for u in urls if u != self._healthy_url]
+
+        for url in urls:
             try:
                 server = ldap3.Server(
-                    url.rstrip("/"),
+                    url,
                     get_info=ldap3.ALL,
                     connect_timeout=5,
                 )
@@ -305,10 +347,13 @@ class LDAPAuthService:
                 conn.unbind()
 
                 if user_info.dn:
+                    self._healthy_url = url
                     return user_info
 
             except LDAPException as e:
                 SYLogger.warning(f"[LDAP] 搜索连接 {url} 失败: {e}")
+                if url == self._healthy_url:
+                    self._healthy_url = None
                 continue
 
         return None

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/src/sycommon/database/async_database_service.py

@@ -28,7 +28,11 @@ class AsyncDatabaseService(metaclass=SingletonMeta):
     _setup_args = None
 
     @staticmethod
-    async def setup_database(config: dict, shareConfigKey: str):
+    async def setup_database(config: dict, shareConfigKey: str = None):
+        # shareConfigKey 缺省时取 app.yaml 的 Name 服务名
+        # (对齐 Services.plugins 的 database_service 默认值)
+        if not shareConfigKey:
+            shareConfigKey = (config or {}).get('Name', '')
         common = NacosService(config).share_configs.get(shareConfigKey, {})
         if common and common.get('spring', {}).get('datasource', None):
             databaseConfig = common.get('spring', {}).get('datasource', None)

{sycommon_python_lib-0.2.10a4 → sycommon_python_lib-0.2.10b6}/src/sycommon/database/database_service.py

@@ -15,7 +15,11 @@ class DatabaseService(metaclass=SingletonMeta):
     _setup_args = None
 
     @staticmethod
-    def setup_database(config: dict, shareConfigKey: str):
+    def setup_database(config: dict, shareConfigKey: str = None):
+        # shareConfigKey 缺省时取 app.yaml 的 Name 服务名
+        # (对齐 Services.plugins 的 database_service 默认值)
+        if not shareConfigKey:
+            shareConfigKey = (config or {}).get('Name', '')
         common = NacosService(config).share_configs.get(shareConfigKey, {})
         if common and common.get('spring', {}).get('datasource', None):
             databaseConfig = common.get('spring', {}).get('datasource', None)

sycommon_python_lib-0.2.10b6/src/sycommon/logging/bounded_kafka_sink.py

@@ -0,0 +1,156 @@
+"""有界 Kafka sink 包装器 —— 治 loguru enqueue=True 无界队列 OOM(两项目共用)。
+
+背景(9/10 生产内存排查):KafkaLogger 的 Kafka sink 以 `logger.add(sink,
+enqueue=True, level="DEBUG", diagnose=True)` 接入。loguru enqueue 的内部队列
+是 queue.Queue()(无界),消费线程停滞(Kafka broker 抖动 / max_block_ms 阻塞 /
+重建窗口)时队列无限堆积 —— kafka_log.py 注释自认"内存无上限直到 OOM,且无
+任何信号"。生产真实日志量 23.7 万条/12h,堆积几小时即 GB 级。
+
+stdout 侧已有 BoundedStdoutSink 三级防线;本类对 Kafka sink 做同款有界化:
+
+  深度 < soft(默认 65536):常态,零等待零丢失
+  满(hard 默认 262144,≈256MB 级文本):put 限时等待 block_wait(默认 2s)
+  超时:丢弃保进程,周期性把丢弃计数打到 stdout(可观测,不再"无任何信号")
+
+与 loguru 集成:logger.add(sink=BoundedKafkaSink(inner_sink), enqueue=False)
+—— 不用 loguru 的 enqueue(它无界),自带后台线程消费 inner_sink.write。
+
+write() 切断 loguru Message 的 .record 活引用(record 挂调用方栈帧,diagnose
+时可达局部大对象)——队列只存纯文本,积压内存 = 文本体量。
+"""
+import queue as _queue
+import sys
+import threading
+import time
+from datetime import datetime
+
+
+class BoundedKafkaSink:
+    """线程安全的有界 Kafka sink 包装器;write 永不直接调用底层 Kafka 发送。"""
+
+    def __init__(self, inner_sink, soft_limit: int = 65536,
+                 hard_limit: int = 262144, block_wait: float = 2.0):
+        self._inner = inner_sink
+        self._q: _queue.Queue = _queue.Queue(maxsize=hard_limit)
+        self._soft = soft_limit
+        self._wait = block_wait
+        self._closed = False
+        self._warned = False
+        self._drops = 0
+        self._drop_report_ts = 0.0
+        self._t = threading.Thread(
+            target=self._loop, name="sycommon-kafka-sink", daemon=True)
+        self._t.start()
+
+    # ---- loguru 契约 ----
+    def write(self, message):
+        """带超时入队;队列只存纯文本,不存带栈帧引用的 Message。"""
+        if self._closed:
+            return
+        text = str(message)
+        try:
+            self._q.put(text, timeout=0)
+            return
+        except _queue.Full:
+            pass
+        try:
+            self._q.put(text, timeout=self._wait)
+        except _queue.Full:
+            self._drops += 1
+            self._report_drops()
+
+    def flush(self):
+        pass  # 后台线程持续刷;loguru 契约要求有 flush
+
+    def stop(self, drain_timeout: float = 5.0):
+        """停后台线程;先排空缓冲再退。"""
+        self._closed = True
+        while True:
+            try:
+                self._q.put_nowait(None)
+                break
+            except _queue.Full:
+                try:
+                    self._q.get_nowait()
+                except _queue.Empty:
+                    pass
+        self._t.join(timeout=drain_timeout)
+
+    # ---- 监控 ----
+    @property
+    def drops(self) -> int:
+        """累计丢弃条数。"""
+        return self._drops
+
+    @property
+    def depth(self) -> int:
+        """当前积压深度(近似)。"""
+        return self._q.qsize()
+
+    # ---- 内部 ----
+    def _report_drops(self):
+        """丢弃计数节流上报(60s 一次),补上'无任何信号'的缺口。"""
+        now = time.monotonic()
+        if now - self._drop_report_ts < 60:
+            return
+        self._drop_report_ts = now
+        try:
+            sys.__stderr__.write(
+                f"[logs] kafka sink dropped {self._drops} msgs "
+                f"(consumer stalled? depth={self._q.qsize()})\n")
+        except Exception:
+            pass
+
+    def _loop(self):
+        err = sys.__stderr__
+        while True:
+            m = self._q.get()
+            if m is None:
+                return
+            try:
+                # inner 是原 KafkaSink.write(loguru Message 兼容对象)。
+                # 传 str(已是格式化文本),KafkaSink.write 只读 .record —— 包一个
+                # 轻量 str 子类带空 record,避免它 AttributeError。
+                self._inner.write(_StrWithRecord(m))
+                depth = self._q.qsize()
+                if depth >= self._soft and not self._warned and err:
+                    self._warned = True
+                    err.write(
+                        f"[logs] kafka backlog {depth} msgs "
+                        f"(>= soft limit {self._soft}); consumer stalled?\n")
+                elif depth < self._soft // 2:
+                    self._warned = False
+            except Exception:
+                # 底层 write 自身吞异常(KafkaSink 内部全 try),这里兜底防线程死
+                time.sleep(0.01)
+
+
+class _StrWithRecord(str):
+    """str 子类,挂一个空 record —— 兼容 KafkaSink.write 读 message.record。
+
+    KafkaSink.write 实际只消费:extra(_trace_id/_thread_info)、exception(判
+    .type)、file.name、name、time.strftime、level.name、message、function、line。
+    这里给全兜底值;时间用消费时刻(loguru record 本来也是写入时刻)。
+    """
+
+    @property
+    def record(self):
+        return {
+            "extra": {},
+            "message": str(self),
+            "level": _Rec(name="INFO"),
+            "exception": None,
+            "time": datetime.now(),
+            "file": _Rec(name="kafka"),
+            "name": "",
+            "function": "",
+            "line": 0,
+            "thread": None,
+        }
+
+
+class _Rec:
+    """极简属性袋(level.name / file.name / time.strftime 用)。"""
+
+    def __init__(self, name):
+        self.name = name