p2pstore 0.1.7.dev1__tar.gz → 0.1.7.dev2__tar.gz

{p2pstore-0.1.7.dev1/src/p2pstore.egg-info → p2pstore-0.1.7.dev2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: p2pstore
-Version: 0.1.7.dev1
-Summary: P2P Store - 基于 RDMA 的高性能分布式数据传输系统
+Version: 0.1.7.dev2
+Summary: P2P Store - 基于 RDMA 的高性能分布式数据传输系统,更改 ttl 设置
 Author: Paddle
 License-Expression: Apache-2.0
 Keywords: p2p,rdma,distributed,tensor,transfer

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "p2pstore"
-version = "0.1.7.dev1"
-description = "P2P Store - 基于 RDMA 的高性能分布式数据传输系统"
+version = "0.1.7.dev2"
+description = "P2P Store - 基于 RDMA 的高性能分布式数据传输系统,更改 ttl 设置"
 readme = "README.md"
 license = "Apache-2.0"
 requires-python = ">=3.10"

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2}/src/p2pstore/metadata/__init__.py

@@ -22,9 +22,7 @@ def create_metadata_client(
     metadata_type = (config.metadata_type or "etcd").lower()
 
     if metadata_type != "etcd":
-        raise ValueError(
-            f"不支持的元数据类型: {metadata_type}。当前版本仅支持 Etcd。"
-        )
+        raise ValueError(f"不支持的元数据类型: {metadata_type}。当前版本仅支持 Etcd。")
 
     if EtcdMetadataClient is None:
         raise ImportError("请先安装 etcd3 库: pip install etcd3")
@@ -38,6 +36,7 @@ def create_metadata_client(
         client_id=client_id,
         registered_keys=registered_keys,
         enable_watch=config.enable_watch,
+        ttl=config.etcd_lease_ttl,
     )
 
 

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2}/src/p2pstore/metadata/etcd_client.py

@@ -9,6 +9,7 @@ Etcd Metadata Client 实现模块.
 
 import json
 import os
+import random
 import threading
 import time
 from urllib.parse import urlparse
@@ -38,6 +39,7 @@ class EtcdMetadataClient(MetadataClient):
         client_id: str,
         registered_keys: dict[str, int] | None = None,
         enable_watch: bool = True,
+        ttl: int = 3600,
     ):
         self.logger = LoggerManager.get_logger("etcd-metadata-client")
         self.local_ip = local_ip
@@ -49,6 +51,7 @@ class EtcdMetadataClient(MetadataClient):
         self._registered_keys = registered_keys if registered_keys is not None else {}
         self.etcd_endpoints = etcd_endpoints
         self.enable_watch = enable_watch  # 保存配置
+        self.ttl = ttl  # 租约 TTL
 
         # 连接 Etcd (目前简单取第一个地址，生产环境可做高可用轮询)
         self._client = self._connect_etcd()
@@ -89,11 +92,11 @@ class EtcdMetadataClient(MetadataClient):
 
             # 增加 gRPC 消息大小限制，解决大量文件时 list_files 超限问题
             # grpc_message:"CLIENT: Received message larger than max (42587779 vs. 4194304)
-            # 默认 4MB -> 512MB (支持 ~200 万条文件元数据，每条 ~250 bytes)
+            # 默认 4MB -> 1GB (与服务端 --max-request-bytes 保持一致)
             # 注意：过大会增加内存占用和传输时间，建议根据实际需求调整
             grpc_options = [
-                ("grpc.max_send_message_length", 512 * 1024 * 1024),
-                ("grpc.max_receive_message_length", 512 * 1024 * 1024),
+                ("grpc.max_send_message_length", 1024 * 1024 * 1024),
+                ("grpc.max_receive_message_length", 1024 * 1024 * 1024),
                 # 添加 gRPC 超时，防止网络分区时永久阻塞
                 ("grpc.keepalive_time_ms", 10000),  # 10秒发送keepalive
                 ("grpc.keepalive_timeout_ms", 5000),  # 5秒keepalive超时
@@ -297,7 +300,7 @@ class EtcdMetadataClient(MetadataClient):
                     return False
 
                 # 重新创建 Lease
-                self.lease = self._client.lease(ttl=60)
+                self.lease = self._client.lease(ttl=self.ttl)
                 self.lease_id = getattr(self.lease, "id", None)  # 防御性获取
 
                 if not self.lease_id:
@@ -335,8 +338,8 @@ class EtcdMetadataClient(MetadataClient):
             # 保存 host 地址，供 close() 使用
             self.local_host = host
 
-            # 创建 60秒 TTL 的租约
-            self.lease = self._client.lease(ttl=60)
+            # 创建租约
+            self.lease = self._client.lease(ttl=self.ttl)
             # self.lease_id = self.lease.id
             self.lease_id = getattr(self.lease, "id", None)  # 统一使用防御性获取
             if not self.lease_id:
@@ -347,7 +350,12 @@ class EtcdMetadataClient(MetadataClient):
 
             # 写入并绑定租约（全局timeout已在client初始化时设置）
             self._client.put(key, value, lease=self.lease)
-            self.logger.info("Provider 注册成功 (Lease ID: %x)", self.lease_id)
+            self.logger.info(
+                "Provider 注册成功: host=%s, Lease ID=0x%x, TTL=%ds",
+                host,
+                self.lease_id,
+                self.ttl,
+            )
 
             # 启动自动续租线程 (etcd3 库并未内置自动后台刷新，需要手动或使用 refresh)
             # 实际上 python-etcd3 的 lease 对象没有自动 refresh thread，这里简单实现一个
@@ -381,7 +389,27 @@ class EtcdMetadataClient(MetadataClient):
             try:
                 self.lease.refresh()
                 consecutive_failures = 0  # 成功后重置失败计数
-                time.sleep(10)  # 每 10 秒刷新一次（TTL 60s）
+
+                # [新增日志] 打印续租成功状态
+                # 如果 TTL 较长(>60s) 或 调试模式，打印 INFO 日志，让用户安心
+                # 对于短 TTL，避免日志刷屏，仅在 DEBUG 级别打印
+                if self.ttl > 60:
+                    self.logger.info(
+                        "[ETCD-HEARTBEAT] 续租成功: lease_id=0x%x, ttl=%ds, host=%s, status=Active",
+                        self.lease_id or 0,
+                        self.ttl,
+                        self.local_host,
+                    )
+                else:
+                    self.logger.debug(
+                        "[ETCD-HEARTBEAT] 续租成功: lease_id=0x%x", self.lease_id
+                    )
+
+                # 动态调整刷新间隔，保证在 TTL 过期前至少刷新 3 次
+                base_interval = max(1, self.ttl / 3)
+                # 添加 10% 的随机抖动，防止太多 client 同时发起请求 (Thundering Herd)
+                jitter = base_interval * 0.1 * (random.random() * 2 - 1)
+                time.sleep(base_interval + jitter)
 
             except Exception as e:
                 consecutive_failures += 1
@@ -491,7 +519,8 @@ class EtcdMetadataClient(MetadataClient):
         key = f"{self.PREFIX_FILE}{file_key}"
 
         self.logger.debug(
-            "[ETCD] 准备删除文件: key=%s",
+            "[ETCD] 准备删除文件: client_id=%s, key=%s",
+            self.client_id,
             file_key,
         )
 
@@ -500,17 +529,25 @@ class EtcdMetadataClient(MetadataClient):
             success = self._client.delete(key)
             if success:
                 self.logger.info(
-                    "[ETCD] Etcd 删除成功: key=%s, 等待 Watch 广播 DeleteEvent",
+                    "[ETCD] Etcd 删除成功: client_id=%s, key=%s, 等待 Watch 广播 DeleteEvent",
+                    self.client_id,
                     file_key,
                 )
                 return True
             else:
                 self.logger.warning(
-                    "[ETCD] Etcd 删除返回 False: key=%s (可能已不存在)", file_key
+                    "[ETCD] Etcd 删除返回 False: client_id=%s, key=%s (可能已不存在)",
+                    self.client_id,
+                    file_key,
                 )
             return False
         except Exception as e:
-            self.logger.error("[ETCD] 删除文件异常: key=%s, error=%s", file_key, e)
+            self.logger.error(
+                "[ETCD] 删除文件异常: client_id=%s, key=%s, error=%s",
+                self.client_id,
+                file_key,
+                e,
+            )
             return False
 
     def delete_prefix(self, prefix: str) -> bool:
@@ -544,7 +581,8 @@ class EtcdMetadataClient(MetadataClient):
                 last_deleted = getattr(response, "deleted", None)
                 elapsed = time.perf_counter() - start
                 self.logger.info(
-                    "[ETCD] delete_prefix 返回: prefix=%s, attempt=%d/%d, deleted=%s, elapsed=%.3fs",
+                    "[ETCD] delete_prefix 返回: client_id=%s, prefix=%s, attempt=%d/%d, deleted=%s, elapsed=%.3fs",
+                    self.client_id,
                     prefix,
                     attempt,
                     max_attempts,
@@ -554,7 +592,8 @@ class EtcdMetadataClient(MetadataClient):
             except Exception as e:
                 last_error = e
                 self.logger.warning(
-                    "[ETCD] delete_prefix 异常: prefix=%s, attempt=%d/%d, error=%s",
+                    "[ETCD] delete_prefix 异常: client_id=%s, prefix=%s, attempt=%d/%d, error=%s",
+                    self.client_id,
                     prefix,
                     attempt,
                     max_attempts,
@@ -704,7 +743,11 @@ class EtcdMetadataClient(MetadataClient):
             keys_to_clear = list(files.keys())
             count = len(keys_to_clear)
 
-            self.logger.info("[ETCD] 开始清空文件: total=%d (查询自 Etcd)", count)
+            self.logger.info(
+                "[ETCD] 开始清空文件: client_id=%s, total=%d (查询自 Etcd)",
+                self.client_id,
+                count,
+            )
 
             # 逐个删除以触发 Watch DeleteEvent
             failed_keys = []

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2}/src/p2pstore/utils/config.py

@@ -48,9 +48,17 @@ class P2PConfig:
     retry_interval: int = 5
     log_name: str | None = None
     enable_watch: bool = True  # 是否启用 Etcd Watch (Consumer get 节点设为 False)
+    etcd_lease_ttl: int = 3600  # Etcd 租约 TTL (秒)，默认 1小时
 
     def __post_init__(self) -> None:
         """初始化后处理."""
+        # 优先使用环境变量中的 TTL 设置
+        if os.getenv("P2P_ETCD_LEASE_TTL"):
+            try:
+                self.etcd_lease_ttl = int(os.environ["P2P_ETCD_LEASE_TTL"])
+            except ValueError:
+                pass
+
         # 自动填充 local_host
         if not self.local_host:
             self.local_host = os.getenv("POD_IP", "").strip() or "127.0.0.1"

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2}/src/p2pstore/utils/logger.py

@@ -206,7 +206,8 @@ class LoggerManager:
         else:
             fmt = f"%(asctime)s - {client_prefix}%(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s"
 
-        formatter = logging.Formatter(fmt, datefmt="%Y-%m-%d %H:%M:%S")
+        # 不指定 datefmt，默认格式为 "%Y-%m-%d %H:%M:%S,333" (包含毫秒)
+        formatter = logging.Formatter(fmt)
 
         log_file = cls._get_log_dir() / f"{name}.log"
         print(

{p2pstore-0.1.7.dev1 → p2pstore-0.1.7.dev2/src/p2pstore.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: p2pstore
-Version: 0.1.7.dev1
-Summary: P2P Store - 基于 RDMA 的高性能分布式数据传输系统
+Version: 0.1.7.dev2
+Summary: P2P Store - 基于 RDMA 的高性能分布式数据传输系统,更改 ttl 设置
 Author: Paddle
 License-Expression: Apache-2.0
 Keywords: p2p,rdma,distributed,tensor,transfer