sycommon-python-lib 0.2.6a2__tar.gz → 0.2.6a3__tar.gz

{sycommon_python_lib-0.2.6a2 → sycommon_python_lib-0.2.6a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.6a2
+Version: 0.2.6a3
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.6a2 → sycommon_python_lib-0.2.6a3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.6a2"
+version = "0.2.6a3"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

sycommon_python_lib-0.2.6a3/src/sycommon/agent/sandbox/consistent_hash.py

@@ -0,0 +1,175 @@
+"""
+一致性哈希沙箱路由
+
+虚拟槽位层：将 0-10000 的虚拟槽位通过一致性哈希环映射到真实沙箱实例。
+用户到虚拟槽位的映射永久固定（只依赖 user_id），增减真实沙箱只影响槽位到实例的映射。
+
+架构:
+┌────────────────────────────────────────────────────────────┐
+│                    用户请求                                 │
+│                                                            │
+│  user_id ──MD5──▶ 虚拟槽位号 (0-10000)                     │
+│                          │                                 │
+│                          ▼                                 │
+│  ┌─────────────────────────────────────────────────────┐   │
+│  │              一致性哈希环                             │   │
+│  │                                                     │   │
+│  │  每个真实实例在环上占 150 个虚拟节点                   │   │
+│  │  槽位号顺时针找到最近的虚拟节点 → 真实实例             │   │
+│  └─────────────────────────────────────────────────────┘   │
+│                          │                                 │
+│                          ▼                                 │
+│  ┌─────────────────────────────────────────────────────┐   │
+│  │              真实沙箱实例                             │   │
+│  │    192.168.1.100:8080  (实例1)                       │   │
+│  │    192.168.1.101:8080  (实例2)                       │   │
+│  │    192.168.1.102:8080  (实例3)                       │   │
+│  └─────────────────────────────────────────────────────┘   │
+│                                                            │
+│  增减实例时：仅相邻槽位重新分配，绝大部分用户不受影响       │
+└────────────────────────────────────────────────────────────┘
+"""
+
+import hashlib
+from bisect import bisect_right
+from typing import Optional
+
+from sycommon.logging.kafka_log import SYLogger
+
+# 虚拟槽位数量：0 到 VIRTUAL_SLOT_COUNT-1
+VIRTUAL_SLOT_COUNT = 10001  # 0-10000
+
+# 每个真实实例在哈希环上的虚拟节点数（越多分布越均匀）
+DEFAULT_VIRTUAL_NODES_PER_INSTANCE = 500
+
+
+class ConsistentHashMap:
+    """一致性哈希环，将虚拟槽位号映射到真实沙箱实例。
+
+    使用虚拟节点技术保证负载均衡均匀。
+    线程安全：build() 后 ring 为不可变结构，get_instance() 无需加锁。
+    """
+
+    def __init__(self, virtual_nodes_per_instance: int = DEFAULT_VIRTUAL_NODES_PER_INSTANCE):
+        self._ring: list[int] = []           # 排序的哈希环（虚拟节点的哈希值）
+        self._node_map: dict[int, str] = {}  # ring_hash -> instance_key (ip:port)
+        self._virtual_nodes = virtual_nodes_per_instance
+
+    def build(self, instances: list[dict]) -> None:
+        """从 Nacos 实例列表构建一致性哈希环。
+
+        Args:
+            instances: Nacos 实例列表，每个元素包含 'ip' 和 'port' 字段
+        """
+        self._ring = []
+        self._node_map = {}
+
+        for ins in instances:
+            key = f"{ins['ip']}:{ins['port']}"
+            for i in range(self._virtual_nodes):
+                # 虚拟节点标识
+                vnode_key = f"{key}#vn{i}"
+                h = self._hash(vnode_key)
+                self._ring.append(h)
+                self._node_map[h] = key
+
+        self._ring.sort()
+        SYLogger.info(
+            f"[ConsistentHash] 哈希环构建完成: "
+            f"{len(instances)} 个实例, {len(self._ring)} 个虚拟节点"
+        )
+
+    def get_instance(self, slot: int) -> Optional[str]:
+        """虚拟槽位号 → 最近的真实实例 key (ip:port)。
+
+        在哈希环上顺时针查找最近的虚拟节点。
+        如果 slot 对应的位置之后没有节点，回到环首。
+
+        Args:
+            slot: 虚拟槽位号 (0 ~ VIRTUAL_SLOT_COUNT-1)
+
+        Returns:
+            实例 key "ip:port"，环为空时返回 None
+        """
+        if not self._ring:
+            return None
+
+        # 将 slot 映射到哈希环上的一个位置
+        h = self._hash(str(slot))
+        # 顺时针找到第一个 >= h 的节点
+        idx = bisect_right(self._ring, h)
+        if idx >= len(self._ring):
+            idx = 0  # 回到环首
+        return self._node_map[self._ring[idx]]
+
+    @staticmethod
+    def _hash(key: str) -> int:
+        """将字符串映射到 0 ~ 2^64-1 的哈希空间"""
+        return int(hashlib.md5(key.encode()).hexdigest(), 16) % (2 ** 64)
+
+
+def user_to_slot(user_id: str) -> int:
+    """用户 ID → 固定虚拟槽位号。
+
+    同一 user_id 永远返回相同的槽位号。
+    槽位号范围: 0 ~ VIRTUAL_SLOT_COUNT-1
+
+    Args:
+        user_id: 用户 ID
+
+    Returns:
+        虚拟槽位号 (int)
+    """
+    hash_value = int(hashlib.md5(user_id.encode()).hexdigest(), 16)
+    return hash_value % VIRTUAL_SLOT_COUNT
+
+
+def resolve_user_instance(user_id: str, instances: list[dict]) -> dict:
+    """用户 ID + 可用实例列表 → 目标真实实例。
+
+    完整路由流程:
+    1. user_id → 虚拟槽位号（永久固定）
+    2. 构建一致性哈希环
+    3. 槽位号 → 真实实例
+
+    Args:
+        user_id: 用户 ID
+        instances: Nacos 可用实例列表，每个元素包含 'ip' 和 'port'
+
+    Returns:
+        目标实例 dict（含 'ip', 'port' 等字段）
+
+    Raises:
+        RuntimeError: 实例列表为空
+    """
+    if not instances:
+        raise RuntimeError("实例列表为空，无法路由")
+
+    # 单实例直接返回，无需哈希
+    if len(instances) == 1:
+        return instances[0]
+
+    slot = user_to_slot(user_id)
+    ring = ConsistentHashMap()
+    ring.build(instances)
+    target_key = ring.get_instance(slot)
+
+    if target_key is None:
+        # 兜底：环构建失败时取第一个实例
+        SYLogger.warning(
+            f"[ConsistentHash] 哈希环为空，回退到第一个实例")
+        return instances[0]
+
+    # 查找目标实例
+    for ins in instances:
+        if f"{ins['ip']}:{ins['port']}" == target_key:
+            SYLogger.info(
+                f"[ConsistentHash] 路由: user_id={user_id} -> "
+                f"slot={slot} -> instance={target_key}"
+            )
+            return ins
+
+    # 未匹配（理论上不会发生），兜底
+    SYLogger.warning(
+        f"[ConsistentHash] 未找到匹配实例 key={target_key}，回退到第一个实例")
+    return instances[0]

{sycommon_python_lib-0.2.6a2 → sycommon_python_lib-0.2.6a3}/src/sycommon/agent/sandbox/http_sandbox_backend.py

@@ -316,13 +316,10 @@ class HTTPSandboxBackend(FileOperationsMixin, SandboxBackendProtocol):
         if not instances:
             raise RuntimeError(f"服务 [{service_name}] 无可用实例（已过滤非所有者本地沙箱）")
 
-        # 基于用户ID的负载均衡：使用哈希选择实例
+        # 基于用户ID的一致性哈希负载均衡：用户固定映射到虚拟槽位，槽位映射到真实实例
         if load_balance and len(instances) > 1:
-            hash_value = int(hashlib.md5(user_id.encode()).hexdigest(), 16)
-            instance_index = hash_value % len(instances)
-            instance = instances[instance_index]
-            SYLogger.info(
-                f"[Sandbox] 负载均衡: user_id={user_id} -> instance[{instance_index}/{len(instances)}]")
+            from sycommon.agent.sandbox.consistent_hash import resolve_user_instance
+            instance = resolve_user_instance(user_id, instances)
         else:
             instance = instances[0]
 

sycommon_python_lib-0.2.6a3/src/sycommon/tests/test_consistent_hash.py

@@ -0,0 +1,279 @@
+"""
+一致性哈希沙箱路由 - 验证测试
+
+验证内容:
+1. 用户粘性：同一 user_id 多次调用返回相同实例
+2. 增实例：新增沙箱后只有部分用户迁移
+3. 删实例：删除沙箱后受影响用户被分散
+4. 均匀分布：10000 虚拟槽位均匀分配到各实例
+5. 虚拟槽位固定：user_id 到槽位号永久不变
+"""
+
+import sys
+import os
+import random
+import hashlib
+from collections import Counter
+
+# 抑制 SYLogger 日志
+os.environ["LOG_LEVEL"] = "ERROR"
+
+# 添加路径
+sys.path.insert(0, "/Users/osulcode.xiao/miniconda3/lib/python3.11/site-packages")
+
+from sycommon.agent.sandbox.consistent_hash import (
+    ConsistentHashMap,
+    VIRTUAL_SLOT_COUNT,
+    user_to_slot,
+    resolve_user_instance,
+)
+
+
+def make_instances(n: int) -> list[dict]:
+    """生成 n 个模拟沙箱实例"""
+    return [{"ip": f"10.0.0.{i+1}", "port": 8080} for i in range(n)]
+
+
+def instance_key(ins: dict) -> str:
+    return f"{ins['ip']}:{ins['port']}"
+
+
+def test_slot_stability():
+    """测试 1: user_id 到虚拟槽位的映射永久固定"""
+    print("=" * 60)
+    print("测试 1: user_id → 虚拟槽位 永久固定")
+    print("=" * 60)
+
+    users = [f"user_{i}" for i in range(100)]
+    slots_round1 = {u: user_to_slot(u) for u in users}
+    slots_round2 = {u: user_to_slot(u) for u in users}
+
+    assert slots_round1 == slots_round2, "槽位映射不稳定！"
+    print(f"✅ {len(users)} 个用户的槽位映射在两次调用中完全一致")
+
+    # 验证槽位范围
+    for u, s in slots_round1.items():
+        assert 0 <= s < VIRTUAL_SLOT_COUNT, f"槽位 {s} 超出范围"
+    print(f"✅ 所有槽位号在 [0, {VIRTUAL_SLOT_COUNT-1}] 范围内")
+
+    # 显示几个示例
+    print("\n示例映射:")
+    for u in users[:5]:
+        print(f"  {u} → 槽位 {user_to_slot(u)}")
+    print()
+
+
+def test_user_stickiness():
+    """测试 2: 同一用户在相同实例列表下始终路由到同一实例"""
+    print("=" * 60)
+    print("测试 2: 用户粘性 - 相同实例列表，路由不变")
+    print("=" * 60)
+
+    instances = make_instances(3)
+    users = [f"user_{i}" for i in range(200)]
+
+    results = []
+    for _ in range(5):  # 调用 5 次
+        round_result = {u: instance_key(resolve_user_instance(u, instances)) for u in users}
+        results.append(round_result)
+
+    # 所有轮次的结果应该完全一致
+    for i in range(1, len(results)):
+        assert results[0] == results[i], f"第 {i} 轮结果不一致！"
+
+    print(f"✅ {len(users)} 个用户在 5 次调用中路由完全一致")
+
+    # 统计分布
+    counter = Counter(results[0].values())
+    print(f"\n分布: {dict(counter)}")
+    print()
+
+
+def test_add_instance_minimal_migration():
+    """测试 3: 新增实例时最少迁移"""
+    print("=" * 60)
+    print("测试 3: 新增沙箱 - 只有部分用户迁移")
+    print("=" * 60)
+
+    users = [f"user_{i}" for i in range(1000)]
+
+    # 3 个实例
+    instances_3 = make_instances(3)
+    mapping_3 = {u: instance_key(resolve_user_instance(u, instances_3)) for u in users}
+
+    # 扩到 4 个实例
+    instances_4 = make_instances(4)
+    mapping_4 = {u: instance_key(resolve_user_instance(u, instances_4)) for u in users}
+
+    # 计算迁移数
+    migrated = sum(1 for u in users if mapping_3[u] != mapping_4[u])
+    migration_rate = migrated / len(users) * 100
+
+    print(f"3 → 4 实例: {migrated}/{len(users)} 用户迁移 ({migration_rate:.1f}%)")
+    print(f"理论最优迁移率: ~{100/4:.1f}% (1/4 的用户应被重新分配到新实例)")
+
+    # 迁移率应接近 1/N（新增第 4 个实例，理论 ~25% 的用户迁移）
+    assert migration_rate < 40, f"迁移率 {migration_rate:.1f}% 过高！"
+    print(f"✅ 迁移率 {migration_rate:.1f}% 在合理范围内")
+
+    # 验证未迁移的用户保持不变
+    unchanged = sum(1 for u in users if mapping_3[u] == mapping_4[u])
+    print(f"✅ {unchanged}/{len(users)} 用户保持原有沙箱不变")
+
+    # 扩到 5 个实例
+    instances_5 = make_instances(5)
+    mapping_5 = {u: instance_key(resolve_user_instance(u, instances_5)) for u in users}
+    migrated_4_to_5 = sum(1 for u in users if mapping_4[u] != mapping_5[u])
+    rate_4_to_5 = migrated_4_to_5 / len(users) * 100
+    print(f"4 → 5 实例: {migrated_4_to_5}/{len(users)} 用户迁移 ({rate_4_to_5:.1f}%)")
+    assert rate_4_to_5 < 30, f"迁移率 {rate_4_to_5:.1f}% 过高！"
+    print(f"✅ 迁移率 {rate_4_to_5:.1f}% 在合理范围内")
+    print()
+
+
+def test_remove_instance():
+    """测试 4: 删除实例时受影响用户被分散"""
+    print("=" * 60)
+    print("测试 4: 删除沙箱 - 受影响用户被分散到其他实例")
+    print("=" * 60)
+
+    users = [f"user_{i}" for i in range(1000)]
+
+    # 4 个实例
+    instances_4 = make_instances(4)
+    mapping_4 = {u: instance_key(resolve_user_instance(u, instances_4)) for u in users}
+
+    # 减到 3 个（删除最后一个）
+    instances_3 = make_instances(3)
+    mapping_3 = {u: instance_key(resolve_user_instance(u, instances_3)) for u in users}
+
+    # 被删除实例的用户
+    removed_key = "10.0.0.4:8080"
+    affected_users = [u for u in users if mapping_4[u] == removed_key]
+    print(f"被删除实例 {removed_key} 上有 {len(affected_users)} 个用户")
+
+    # 这些用户在 3 实例下应该被分散到其余实例
+    if affected_users:
+        new_destinations = Counter(mapping_3[u] for u in affected_users)
+        print(f"迁移分布: {dict(new_destinations)}")
+
+        # 不应该有任何用户还路由到已删除的实例
+        for u in affected_users:
+            assert mapping_3[u] != removed_key, f"用户 {u} 仍路由到已删除实例！"
+        print(f"✅ 所有受影响用户已迁移到其他实例")
+
+    # 未受影响的用户保持不变
+    unaffected = [u for u in users if mapping_4[u] != removed_key]
+    unchanged = sum(1 for u in unaffected if mapping_4[u] == mapping_3[u])
+    print(f"✅ {unchanged}/{len(unaffected)} 未受影响用户保持原有沙箱")
+    print()
+
+
+def test_uniform_distribution():
+    """测试 5: 虚拟槽位均匀分布到各实例"""
+    print("=" * 60)
+    print("测试 5: 均匀分布 - 虚拟槽位到实例的分配")
+    print("=" * 60)
+
+    for n_instances in [2, 3, 5, 10]:
+        instances = make_instances(n_instances)
+        ring = ConsistentHashMap()
+        ring.build(instances)
+
+        # 统计所有虚拟槽位的分配
+        counter = Counter()
+        for slot in range(VIRTUAL_SLOT_COUNT):
+            key = ring.get_instance(slot)
+            if key:
+                counter[key] += 1
+
+        total = sum(counter.values())
+        expected = total / n_instances
+        max_deviation = max(abs(v - expected) / expected * 100 for v in counter.values())
+
+        print(f"  {n_instances} 个实例: 每实例期望 {expected:.0f} 槽位, "
+              f"最大偏差 {max_deviation:.1f}%")
+        print(f"    实际分布: {dict(counter)}")
+
+        # 偏差应小于 15%
+        assert max_deviation < 15, f"偏差 {max_deviation:.1f}% 过大！"
+        print(f"  ✅ 偏差在合理范围内")
+    print()
+
+
+def test_large_scale_user_distribution():
+    """测试 6: 大规模用户分布"""
+    print("=" * 60)
+    print("测试 6: 10000 用户在 5 个沙箱上的分布")
+    print("=" * 60)
+
+    instances = make_instances(5)
+    users = [f"user_{i:05d}" for i in range(10000)]
+
+    counter = Counter()
+    for u in users:
+        ins = resolve_user_instance(u, instances)
+        counter[instance_key(ins)] += 1
+
+    expected = len(users) / len(instances)
+    max_deviation = max(abs(v - expected) / expected * 100 for v in counter.values())
+
+    print(f"每实例期望: {expected:.0f} 用户")
+    print(f"实际分布: {dict(counter)}")
+    print(f"最大偏差: {max_deviation:.1f}%")
+    assert max_deviation < 15, f"偏差 {max_deviation:.1f}% 过大！"
+    print(f"✅ 10000 用户在 5 个沙箱上分布均匀")
+    print()
+
+
+def test_comparison_with_old_method():
+    """测试 7: 对比新旧方法在增减实例时的迁移率"""
+    print("=" * 60)
+    print("测试 7: 新(一致性哈希) vs 旧(取模) 方法迁移率对比")
+    print("=" * 60)
+
+    users = [f"user_{i}" for i in range(1000)]
+
+    def old_method(user_id, instances):
+        hash_value = int(hashlib.md5(user_id.encode()).hexdigest(), 16)
+        idx = hash_value % len(instances)
+        return instance_key(instances[idx])
+
+    def new_method(user_id, instances):
+        return instance_key(resolve_user_instance(user_id, instances))
+
+    for old_n, new_n in [(3, 4), (4, 5), (5, 6), (10, 11)]:
+        old_ins = make_instances(old_n)
+        new_ins = make_instances(new_n)
+
+        # 旧方法迁移率
+        old_mapping_before = {u: old_method(u, old_ins) for u in users}
+        old_mapping_after = {u: old_method(u, new_ins) for u in users}
+        old_migrated = sum(1 for u in users if old_mapping_before[u] != old_mapping_after[u])
+
+        # 新方法迁移率
+        new_mapping_before = {u: new_method(u, old_ins) for u in users}
+        new_mapping_after = {u: new_method(u, new_ins) for u in users}
+        new_migrated = sum(1 for u in users if new_mapping_before[u] != new_mapping_after[u])
+
+        print(f"  {old_n} → {new_n} 实例:")
+        print(f"    旧方法(取模): {old_migrated}/{len(users)} 迁移 ({old_migrated/len(users)*100:.1f}%)")
+        print(f"    新方法(一致性): {new_migrated}/{len(users)} 迁移 ({new_migrated/len(users)*100:.1f}%)")
+        print(f"    改善: {old_migrated - new_migrated} 个用户免于迁移")
+    print()
+
+
+if __name__ == "__main__":
+    print("\n🚀 一致性哈希沙箱路由 - 验证测试\n")
+
+    test_slot_stability()
+    test_user_stickiness()
+    test_add_instance_minimal_migration()
+    test_remove_instance()
+    test_uniform_distribution()
+    test_large_scale_user_distribution()
+    test_comparison_with_old_method()
+
+    print("=" * 60)
+    print("🎉 所有测试通过！")
+    print("=" * 60)

{sycommon_python_lib-0.2.6a2 → sycommon_python_lib-0.2.6a3}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.6a2
+Version: 0.2.6a3
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.6a2 → sycommon_python_lib-0.2.6a3}/src/sycommon_python_lib.egg-info/SOURCES.txt

@@ -138,6 +138,7 @@ src/sycommon/agent/mcp/__init__.py
 src/sycommon/agent/mcp/models.py
 src/sycommon/agent/mcp/tool_loader.py
 src/sycommon/agent/sandbox/__init__.py
+src/sycommon/agent/sandbox/consistent_hash.py
 src/sycommon/agent/sandbox/file_ops.py
 src/sycommon/agent/sandbox/http_sandbox_backend.py
 src/sycommon/agent/sandbox/minio_sync.py
@@ -259,6 +260,7 @@ src/sycommon/synacos/nacos_service_discovery.py
 src/sycommon/synacos/nacos_service_registration.py
 src/sycommon/synacos/param.py
 src/sycommon/tests/deep_agent_server.py
+src/sycommon/tests/test_consistent_hash.py
 src/sycommon/tests/test_consumer_loss_rootcause.py
 src/sycommon/tests/test_consumer_recovery.py
 src/sycommon/tests/test_deep_agent.py