sycommon-python-lib 0.2.10a21__tar.gz → 0.2.10a22__tar.gz

{sycommon_python_lib-0.2.10a21 → sycommon_python_lib-0.2.10a22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.10a21
+Version: 0.2.10a22
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.10a21 → sycommon_python_lib-0.2.10a22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.10a21"
+version = "0.2.10a22"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

{sycommon_python_lib-0.2.10a21 → sycommon_python_lib-0.2.10a22}/src/sycommon/alert/monitor.py

@@ -6,12 +6,16 @@ Java GcMonitor(字节码):GC 占比 = ΔGC时间/Δ窗口(GarbageCollectorMXBean
 默认 true;msg "GC占比: %.2f%% (GC时间: %d ms / 统计窗口: %d ms)";标题
 GC致命告警/GC严重告警/GC告警;扫描>500ms warn 耗时过长。
 
-Python 等价物:gc 模块 get_count()/时间差分 — CPython 无 collector 累计时间,
-用 gc 时间占比的近似:显式 gc.collect() 会扰动业务,故用 gc.get_stats +
- tracemalloc 不可用时降级为仅计数占比 — 实用形态:用 gc 回调(since 3.7
- gc.callbacks)累计代际回收次数 + 可选 GC 时长(tracemalloc)。
-保守实现:统计窗口内 gc 次数增长率告警(次数/秒 换算成每 10s 阈值同义),
-配置键与 Java 同名,阈值语义映射为 次数占比(相对窗口)。
+Python 等价物:gc.callbacks(since 3.7)start/stop 时刻差(perf_counter)
+累计单次 GC 真实时长 — 实测 3.11–3.14 的 stop info 均无 duration 字段
+(官方文档亦只定义 generation/collected/uncollectable/candidates),CPython
+无现成单次 GC 计时 API,计时差是唯一可靠来源。累计器差分/窗口 — 与 Java
+ΔgcTime/Δwindow、Go PauseTotalNs 差分同语义。与 Java 的已知差异:Java
+排除 young GC;CPython gen0 单次亚毫秒,全代求和对时间占比影响可忽略,
+且无独立 collector 累计时间可拆。
+【历史缺陷(2026-10 修复)】:旧实现用 collections 计数差分冒充时间占比
+(每秒次数×100 硬套 "%%" 模板),高频小 GC(10 次/10s)即误报 fatal
+"GC占比: 99.70% (GC时间: 10 次)"。
 
 DatasourceMonitor(Druid/Hikari MBean):Python 等价物是 SQLAlchemy engine
 pool.status()(checkedin/checkedout/size/overflow),经 register_db_stats 注入;
@@ -28,11 +32,45 @@ from sycommon.alert.api import critical, fatal, warn
 from sycommon.alert.env import app_name_of_sdk, get_bool, get_float, get_int
 from sycommon.logging.kafka_log import SYLogger
 
-_db_stats_func: Optional[Callable[dict]] = None
+_db_stats_func: Optional[Callable[[], dict]] = None
 _started = False
 
+# gc.callbacks stop 事件的累计毫秒数(单调递增)+ start 时刻(perf_counter)。
+# 回调在持 GIL 下串行执行、全程无 GIL 释放点,读侧只读单个 float — 无需锁;
+# 理论竞态最坏丢一次累加(仅监控口径,可容忍)。
+_gc_time_ms = 0.0
+_gc_last_start = 0.0
+_gc_hook_installed = False
 
-def register_db_stats(func: Callable[dict]) -> None:
+
+def _on_gc_event(phase: str, info: dict) -> None:  # noqa: ARG001 — gc 回调签名
+    """gc.callbacks 回调:start 记时刻;stop 累计单次 GC 真实时长(ms)。
+
+    时长 = stop 时刻 - start 时刻(perf_counter)。实测 3.11–3.14 的 stop
+    info 都没有 duration 字段(官方文档仅定义 generation/collected/
+    uncollectable/candidates),计时差是 CPython 下唯一可靠来源,含回调派发
+    开销(微秒级,方向只高估不低估)。未见过 start 的孤儿 stop 按 0 处理
+    (防 perf_counter 大数污染)。不访问 info 内容 — 字段形态任何版本
+    差异都不会抛异常(gc 回调内抛异常 = 每次 GC 刷 unraisable 栈到
+    stderr,性能事故)。单次成本:2×perf_counter + 浮点加,亚微秒级。
+    """
+    global _gc_time_ms, _gc_last_start
+    if phase == 'start':
+        _gc_last_start = time.perf_counter()
+    else:
+        if _gc_last_start:
+            _gc_time_ms += (time.perf_counter() - _gc_last_start) * 1000.0
+
+
+def _install_gc_hook() -> None:
+    global _gc_hook_installed
+    if _gc_hook_installed:
+        return
+    _gc_hook_installed = True
+    gc.callbacks.append(_on_gc_event)
+
+
+def register_db_stats(func: Callable[[], dict]) -> None:
     """注入连接池饱和度数据源(Java Hikari/Druid MBean 等价物)。
 
     func 返回 {"in_use": int, "max_open": int};max_open<=0 视为未配置。
@@ -67,6 +105,7 @@ def start_monitors() -> None:
     try:
         interval = get_int("sy-alert.gc-monitor.scan-interval-seconds", 10)
         SYLogger.info(f"GC监控开启, 扫描间隔为: {interval} 秒")
+        _install_gc_hook()
         loop.create_task(_gc_monitor_loop(interval))
         if _db_stats_func is not None:
             ds_interval = get_int("sy-alert.datasource.scan-interval-seconds", 10)
@@ -77,47 +116,53 @@ def start_monitors() -> None:
         _started = False
 
 
+def _gc_check_and_alert(delta_ms: int, window_ms: int) -> None:
+    """Java GcMonitor.checkAndAlert 等价:真·时间占比 → 阈值 → 告警。
+
+    delta_ms/window_ms 均为毫秒;rate = ΔGC时间/Δ窗口(与 Java、Go 同语义)。
+    """
+    rate = delta_ms / window_ms
+    msg = (f"GC占比: {rate * 100:.2f}% (GC时间: {delta_ms} ms / "
+           f"统计窗口: {window_ms} ms)")
+    # Java checkAndAlert:每轮先 info "GC监控：{msg}";告警第二参是裸 msg
+    # (无 "GC监控：" 前缀 — 4 轮审核纠偏)。
+    SYLogger.info(f"GC监控：{msg}")
+    falt = get_float("sy-alert.gc-monitor.threshold.falt", 0.5)
+    crit = get_float("sy-alert.gc-monitor.threshold.critical", 0.2)
+    warning = get_float("sy-alert.gc-monitor.threshold.warning", 0.05)
+    if rate < warning:
+        return
+    if get_bool("sy-alert.gc-monitor.enable", True):
+        if rate >= falt:
+            fatal("GC致命告警", msg)
+        elif rate >= crit:
+            critical("GC严重告警", msg)
+        else:
+            warn("GC告警", msg)
+    else:
+        SYLogger.warning(f"GC监控未开启，但当前GC率已达到告警阈值：{msg}")
+
+
 async def _gc_monitor_loop(interval_s: int) -> None:
-    # CPython 无 Java collector 累计时间;等价信号 = 显式统计窗口内的 gc 次数
-    # 占比(每秒 gc 次数 / 1,阈值语义映射同 Java:0.5/0.2/0.05 次/秒)。
+    # 指标 = 窗口内 gc.callbacks stop duration 累计(ms)/窗口(ms) — 与 Java
+    # ΔgcTime/Δwindow、Go PauseTotalNs 差分同语义的真·时间占比。
     # 红线:监控协程顶层兜底 — 任何异常只打日志,任务死亡不带崩宿主。
     try:
-        last_count = gc.get_count()[0] + gc.get_stats()[0]['collections'] * 0
-        base = _total_gc_collections()
+        base = _gc_time_ms
         last_scan = time.monotonic()
         while True:
             await asyncio.sleep(interval_s)
             scan_start = time.monotonic()
             try:
-                cur = _total_gc_collections()
+                cur = _gc_time_ms
                 now = time.monotonic()
                 window_ms = int((now - last_scan) * 1000)
                 if window_ms <= 0:
                     base, last_scan = cur, now
                     continue
-                delta = cur - base
+                delta_ms = int(cur - base)
                 base, last_scan = cur, now
-                rate = delta / (window_ms / 1000.0)
-                msg = (f"GC占比: {rate * 100:.2f}% (GC时间: {delta} 次 / "
-                       f"统计窗口: {window_ms} ms)")
-                _ = last_count
-                # Java checkAndAlert:每轮先 info "GC监控：{msg}";告警第二参是裸 msg
-                # (无 "GC监控：" 前缀 — 4 轮审核纠偏)。
-                SYLogger.info(f"GC监控：{msg}")
-                falt = get_float("sy-alert.gc-monitor.threshold.falt", 0.5)
-                crit = get_float("sy-alert.gc-monitor.threshold.critical", 0.2)
-                warning = get_float("sy-alert.gc-monitor.threshold.warning", 0.05)
-                if rate < warning:
-                    continue
-                if get_bool("sy-alert.gc-monitor.enable", True):
-                    if rate >= falt:
-                        fatal("GC致命告警", msg)
-                    elif rate >= crit:
-                        critical("GC严重告警", msg)
-                    else:
-                        warn("GC告警", msg)
-                else:
-                    SYLogger.warning(f"GC监控未开启，但当前GC率已达到告警阈值：{msg}")
+                _gc_check_and_alert(delta_ms, window_ms)
             except Exception as e:  # noqa: BLE001
                 SYLogger.error(f"GC监控异常: {e}")
             if (time.monotonic() - scan_start) * 1000 > 500:
@@ -128,10 +173,6 @@ async def _gc_monitor_loop(interval_s: int) -> None:
         SYLogger.error(f"[alert] GC监控循环异常退出(不影响宿主服务): {e}")
 
 
-def _total_gc_collections() -> int:
-    return sum(s['collections'] for s in gc.get_stats())
-
-
 async def _db_monitor_loop(interval_s: int) -> None:
     # 红线:监控协程顶层兜底 — 任何异常只打日志,任务死亡不带崩宿主。
     try:

{sycommon_python_lib-0.2.10a21 → sycommon_python_lib-0.2.10a22}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.10a21
+Version: 0.2.10a22
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.10a21 → sycommon_python_lib-0.2.10a22}/src/sycommon_python_lib.egg-info/SOURCES.txt

@@ -389,6 +389,7 @@ src/sycommon_python_lib.egg-info/entry_points.txt
 src/sycommon_python_lib.egg-info/requires.txt
 src/sycommon_python_lib.egg-info/top_level.txt
 tests/test_acp_tool.py
+tests/test_alert_monitor.py
 tests/test_asyncio_logging.py
 tests/test_config_change_notifier.py
 tests/test_frame_alignment.py

sycommon_python_lib-0.2.10a22/tests/test_alert_monitor.py

@@ -0,0 +1,212 @@
+"""test_alert_monitor.py — GC 监控语义修复回归(2026-10)。
+
+历史缺陷:旧实现用 collections 计数差分冒充时间占比(每秒次数×100 硬套
+"%%" 模板),高频小 GC(10 次/10s)即误报 fatal "GC占比: 99.70%
+(GC时间: 10 次)"。修复后指标 = gc.callbacks stop duration 累计 ms/窗口 ms
+真·时间占比,与 Java ΔgcTime/Δwindow、Go PauseTotalNs 差分同语义。
+"""
+import gc
+import os
+import sys
+import time
+from unittest import mock
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+
+os.environ.setdefault("SY_ALERT_UNITTEST", "1")
+
+from sycommon.alert import monitor  # noqa: E402
+
+
+@pytest.fixture(autouse=True)
+def _clean_gc_state():
+    yield
+    monitor._gc_time_ms = 0.0
+    monitor._gc_last_start = 0.0
+    monitor._gc_hook_installed = False
+    for _ in range(gc.callbacks.count(monitor._on_gc_event)):
+        gc.callbacks.remove(monitor._on_gc_event)
+
+
+def test_gc_hook_accumulates_real_duration_ms():
+    """stop 事件按 start/stop 计时差累计 ms;info 内容不参与计算。"""
+    monitor._gc_time_ms = 0.0
+    monitor._on_gc_event('start', {'generation': 0})
+    time.sleep(0.002)  # 2ms
+    monitor._on_gc_event('stop', {'generation': 0, 'collected': 1})
+    monitor._on_gc_event('start', {'generation': 0})
+    time.sleep(0.0005)  # 0.5ms
+    monitor._on_gc_event('stop', {'generation': 0, 'collected': 1})
+    # 计时差 ≥ 睡眠时长,含调度开销;两轮合计 2.5ms + 开销,远小于 10ms
+    assert 2.5 <= monitor._gc_time_ms < 10.0
+
+
+def test_gc_hook_fallback_without_duration_field():
+    """版本回归线:stop info 无 duration 字段(实测 3.11–3.14 均无,官方
+    文档亦只定义 generation/collected/uncollectable/candidates)→ 用
+    start/stop 时刻差计时;孤儿 stop(未见 start)按 0,不抛、不被
+    perf_counter 大数污染。回调不读 info 内容,字段形态差异零影响。"""
+    monitor._gc_time_ms = 0.0
+    monitor._on_gc_event('stop', {'generation': 0, 'collected': 1,
+                                  'uncollectable': 0})  # 孤儿 stop:安全为 0
+    assert monitor._gc_time_ms == 0.0
+    monitor._on_gc_event('start', {'generation': 0})
+    monitor._on_gc_event('stop', {'generation': 0, 'collected': 1,
+                                  'uncollectable': 0})  # 计时差
+    assert monitor._gc_time_ms > 0.0
+    assert monitor._gc_time_ms < 1000.0  # 显然不可能秒级 — 防大数污染
+
+
+def test_gc_hook_install_idempotent():
+    """_install_gc_hook 幂等:多次调用只挂一个回调。"""
+    monitor._install_gc_hook()
+    monitor._install_gc_hook()
+    assert gc.callbacks.count(monitor._on_gc_event) == 1
+
+
+def test_real_gc_updates_counter():
+    """真跑一次 gc.collect:回调链路端到端,计数器只增不减。"""
+    monitor._install_gc_hook()
+    before = monitor._gc_time_ms
+    gc.collect()
+    assert monitor._gc_time_ms >= before
+
+
+def test_no_alert_for_frequent_tiny_gc():
+    """核心回归:10 次亚毫秒 GC/10s 窗口(旧实现 rate=0.997 误报 fatal)
+    → 修复后 2ms/10030ms≈0.02%,低于 warning,不外发任何告警。"""
+    with mock.patch.object(monitor, 'SYLogger'), \
+            mock.patch.object(monitor, 'fatal') as f, \
+            mock.patch.object(monitor, 'critical') as c, \
+            mock.patch.object(monitor, 'warn') as w, \
+            mock.patch.object(monitor, 'get_float',
+                              side_effect=lambda k, d: d), \
+            mock.patch.object(monitor, 'get_bool', return_value=True):
+        # 直接驱动告警判定(等价 loop 内一轮扫描):10 次 0.2ms GC = 2ms。
+        monitor._gc_check_and_alert(delta_ms=2, window_ms=10030)
+    assert not f.called and not c.called and not w.called
+
+
+def test_fatal_on_real_time_ratio():
+    """真实时间占比达标(窗口一半时间在 GC)→ fatal("GC致命告警")。"""
+    with mock.patch.object(monitor, 'SYLogger'), \
+            mock.patch.object(monitor, 'fatal') as f, \
+            mock.patch.object(monitor, 'critical') as c, \
+            mock.patch.object(monitor, 'warn') as w, \
+            mock.patch.object(monitor, 'get_float',
+                              side_effect=lambda k, d: d), \
+            mock.patch.object(monitor, 'get_bool', return_value=True):
+        monitor._gc_check_and_alert(delta_ms=5000, window_ms=10000)
+    f.assert_called_once()
+    assert f.call_args[0][0] == "GC致命告警"
+    assert "GC时间: 5000 ms" in f.call_args[0][1]
+    assert "统计窗口: 10000 ms" in f.call_args[0][1]
+    assert not c.called and not w.called
+
+
+def test_critical_and_warn_tiers():
+    """阈值分层:20%/50% 时间占比 → critical/warn(Java 0.2/0.05 同参)。"""
+    with mock.patch.object(monitor, 'SYLogger'), \
+            mock.patch.object(monitor, 'fatal') as f, \
+            mock.patch.object(monitor, 'critical') as c, \
+            mock.patch.object(monitor, 'warn') as w, \
+            mock.patch.object(monitor, 'get_float',
+                              side_effect=lambda k, d: d), \
+            mock.patch.object(monitor, 'get_bool', return_value=True):
+        monitor._gc_check_and_alert(delta_ms=2000, window_ms=10000)  # 0.2
+        assert c.call_count == 1
+        assert w.call_count == 0 and f.call_count == 0
+        monitor._gc_check_and_alert(delta_ms=500, window_ms=10000)   # 0.05
+        assert w.call_count == 1
+        assert c.call_count == 1 and f.call_count == 0
+
+
+def test_msg_uses_ms_unit_not_count():
+    """msg 单位必须是 ms(时间),不再是「次」(计数)。"""
+    logs: list[str] = []
+
+    class _Log:
+        def info(self, m):
+            logs.append(m)
+
+        def error(self, m):
+            pass
+
+        def warning(self, m):
+            pass
+
+    with mock.patch.object(monitor, 'SYLogger', _Log()), \
+            mock.patch.object(monitor, 'fatal'), \
+            mock.patch.object(monitor, 'critical'), \
+            mock.patch.object(monitor, 'warn'), \
+            mock.patch.object(monitor, 'get_float',
+                              side_effect=lambda k, d: d), \
+            mock.patch.object(monitor, 'get_bool', return_value=True):
+        monitor._gc_check_and_alert(delta_ms=3, window_ms=10000)
+    gc_line = [m for m in logs if m.startswith('GC监控：')]
+    assert gc_line, logs
+    assert "GC时间: 3 ms / 统计窗口: 10000 ms" in gc_line[0]
+    assert "GC占比: 0.03%" in gc_line[0]
+
+
+def test_alert_disabled_only_warns():
+    """sy-alert.gc-monitor.enable=false:超阈值只打 warning,不外发(Java 同)。"""
+    warned: list[str] = []
+
+    class _Log:
+        def info(self, m):
+            pass
+
+        def error(self, m):
+            pass
+
+        def warning(self, m):
+            warned.append(m)
+
+    with mock.patch.object(monitor, 'SYLogger', _Log()), \
+            mock.patch.object(monitor, 'fatal') as f, \
+            mock.patch.object(monitor, 'critical') as c, \
+            mock.patch.object(monitor, 'warn') as w, \
+            mock.patch.object(monitor, 'get_float',
+                              side_effect=lambda k, d: d), \
+            mock.patch.object(monitor, 'get_bool', return_value=False):
+        monitor._gc_check_and_alert(delta_ms=6000, window_ms=10000)
+    assert not f.called and not c.called and not w.called
+    assert any("GC监控未开启" in m for m in warned)
+
+
+def test_monitor_loop_survives_scan_exception():
+    """红线:单轮扫描内异常(配置读取炸)只打日志、循环不死(下轮照常)。
+    隔离在 _gc_monitor_loop 的 try/except — 与 Java checkAndAlert 抛错由
+    扫描包装层捕获同构。"""
+    import asyncio
+
+    errs: list[str] = []
+    flt = mock.Mock(side_effect=RuntimeError("cfg broken"))
+
+    class _Log:
+        def info(self, m):
+            pass
+
+        def error(self, m):
+            errs.append(m)
+
+        def warning(self, m):
+            pass
+
+    async def scenario():
+        with mock.patch.object(monitor, 'SYLogger', _Log()), \
+                mock.patch.object(monitor, 'get_float', flt), \
+                mock.patch.object(monitor, 'get_bool', return_value=True):
+            task = asyncio.ensure_future(monitor._gc_monitor_loop(0.05))
+            await asyncio.sleep(0.25)  # ~4 轮扫描,每轮 get_float 都炸
+            assert not task.done()     # 循环活着,没被异常带死
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+    asyncio.run(scenario())
+    assert flt.call_count >= 2
+    assert any("GC监控异常" in e for e in errs)