proxynt 2.0.58__tar.gz → 2.0.59rc2__tar.gz

{proxynt-2.0.58 → proxynt-2.0.59rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proxynt
-Version: 2.0.58
+Version: 2.0.59rc2
 Summary: UNKNOWN
 Home-page: https://github.com/sazima/proxynt
 License: UNKNOWN

{proxynt-2.0.58 → proxynt-2.0.59rc2}/client/tcp_forward_client.py

@@ -176,7 +176,7 @@ class TcpForwardClient:
         if data.speed_limiter and recv:
             wait_time = data.speed_limiter.acquire(len(recv))
             if wait_time > 0:
-                time.sleep(wait_time)
+                self.socket_event_loop.pause_and_resume_later(each, wait_time)
 
         if self.tunnel_manager:
             if self.tunnel_manager.send_data(connection.uid, recv):
@@ -366,9 +366,15 @@ class TcpForwardClient:
             return
         try:
             s = connection.socket
+            s.settimeout(30)
             s.sendall(msg)
+            s.settimeout(None)
             if not msg:
                 self.close_connection(s)
+        except socket.timeout:
+            LoggerFactory.get_logger().warning('sendall timeout, closing uid: %s' % connection.uid.hex())
+            self.close_connection(s)
+            self.close_remote_socket(connection)
         except Exception:
             LoggerFactory.get_logger().error(traceback.format_exc())
             self.close_remote_socket(connection)

{proxynt-2.0.58 → proxynt-2.0.59rc2}/client/udp_forward_client.py

@@ -188,24 +188,42 @@ class UdpForwardClient:
                 LoggerFactory.get_logger().error(traceback.format_exc())
 
     def _udp_receive_loop(self):
+        import select as _select
         LoggerFactory.get_logger().info('UDP receive thread started')
         while self.running:
             connections = list(self.uid_to_connection.values())
+            if not connections:
+                time.sleep(0.1)
+                continue
+            sock_to_conn = {}
             for conn in connections:
                 try:
                     conn.socket.setblocking(False)
-                    try:
-                        data, addr = conn.socket.recvfrom(65536)
-                        if data:
-                            self._handle_udp_data(conn, data, addr)
-                    except (BlockingIOError, socket.error):
-                        pass
+                    sock_to_conn[conn.socket] = conn
+                except Exception:
+                    pass
+            if not sock_to_conn:
+                time.sleep(0.1)
+                continue
+            try:
+                readable, _, _ = _select.select(list(sock_to_conn.keys()), [], [], 0.5)
+            except (OSError, ValueError):
+                time.sleep(0.1)
+                continue
+            for sock in readable:
+                conn = sock_to_conn.get(sock)
+                if not conn:
+                    continue
+                try:
+                    data, addr = sock.recvfrom(65536)
+                    if data:
+                        self._handle_udp_data(conn, data, addr)
+                except (BlockingIOError, socket.error):
+                    pass
                 except Exception as e:
                     LoggerFactory.get_logger().error('UDP data receive error: %s' % e)
                     LoggerFactory.get_logger().error(traceback.format_exc())
 
-            time.sleep(0.001)
-
     def _handle_udp_data(self, conn: UdpSocketConnection, data: bytes, addr):
         if conn.speed_limiter and data:
             wait_time = conn.speed_limiter.acquire(len(data))

{proxynt-2.0.58 → proxynt-2.0.59rc2}/common/nat_serialization_v2.py

@@ -42,12 +42,11 @@ class NatSerializationV2:
 
         # 处理 data_content，确保可以被 msgpack 序列化
         if data_content is not None:
-            # 复制一份避免修改原数据
-            if compress:
-                serializable_content = dict(data_content) if isinstance(data_content, dict) else data_content
-            else:
-                serializable_content = data_content
-            # 如果有 'data' 字段且需要压缩
+            serializable_content = data_content
+            # 如果有 'data' 字段且需要压缩，才复制一份避免修改原数据
+            if (compress and has_snappy and isinstance(data_content, dict)
+                    and 'data' in data_content and data_content['data']):
+                serializable_content = dict(data_content)
             if isinstance(serializable_content, dict) and 'data' in serializable_content:
                 if compress and has_snappy and serializable_content['data']:
                     original_size = len(serializable_content['data'])

proxynt-2.0.59rc2/common/pool.py

@@ -0,0 +1,96 @@
+import logging
+import socket
+import threading
+import time
+import traceback
+from selectors import DefaultSelector, EVENT_READ
+
+from common.logger_factory import LoggerFactory
+from common.register_append_data import ResisterAppendData
+from constant.system_constant import SystemConstant
+
+"""
+单线程 selector 事件循环, 使用 modify 切换事件掩码
+参考 shadowsocks eventloop 模式
+"""
+
+
+class SelectPool:
+
+    def __init__(self):
+        self.is_running = True
+        self.fileno_to_client = dict()
+        self.selector = DefaultSelector()
+
+    def stop(self):
+        self.is_running = False
+
+    def clear(self):
+        self.fileno_to_client.clear()
+
+    def register(self, s: socket.socket, data: ResisterAppendData):
+        self.fileno_to_client[s.fileno()] = s
+        self.selector.register(s, EVENT_READ, data)
+
+    def unregister(self, s: socket.socket):
+        fileno = -1
+        try:
+            fileno = s.fileno()
+        except Exception:
+            pass
+        if fileno in self.fileno_to_client:
+            self.fileno_to_client.pop(fileno)
+        try:
+            self.selector.unregister(s)
+        except (KeyError, ValueError):
+            pass
+        except OSError:
+            LoggerFactory.get_logger().error(traceback.format_exc())
+
+    def pause_reading(self, s: socket.socket):
+        """暂停监听可读事件 (modify 掩码为 0)"""
+        try:
+            key = self.selector.get_key(s)
+            self.selector.modify(s, 0, key.data)
+        except (KeyError, ValueError):
+            pass
+
+    def resume_reading(self, s: socket.socket):
+        """恢复监听可读事件"""
+        try:
+            key = self.selector.get_key(s)
+            self.selector.modify(s, EVENT_READ, key.data)
+        except (KeyError, ValueError):
+            pass
+
+    def pause_and_resume_later(self, s: socket.socket, delay_time: float):
+        """暂停可读监听, delay_time 秒后恢复"""
+        self.pause_reading(s)
+        threading.Timer(delay_time, self.resume_reading, args=(s,)).start()
+
+    def run(self):
+        while True:
+            if not self.is_running:
+                time.sleep(1)
+                continue
+            try:
+                try:
+                    ready = self.selector.select(timeout=SystemConstant.DEFAULT_TIMEOUT)
+                except OSError:
+                    time.sleep(0.5)
+                    continue
+                for key, mask in ready:
+                    fileno = key.fd
+                    client = self.fileno_to_client.get(fileno)
+                    if client is None:
+                        if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
+                            LoggerFactory.get_logger().debug('fd %s not in fileno_to_client' % fileno)
+                        continue
+                    data: ResisterAppendData = key.data
+                    try:
+                        data.callable_(client, data)
+                    except Exception:
+                        LoggerFactory.get_logger().error(traceback.format_exc())
+            except Exception:
+                LoggerFactory.get_logger().error(traceback.format_exc())
+                time.sleep(1)

{proxynt-2.0.58 → proxynt-2.0.59rc2}/constant/system_constant.py

@@ -15,7 +15,7 @@ class SystemConstant:
 
     COOKIE_EXPIRE_SECONDS = 3600 * 24
 
-    VERSION = '2.0.58'
+    VERSION = '2.0.59.rc2'
 
     GITHUB = 'https://github.com/sazima/proxynt'
 

{proxynt-2.0.58 → proxynt-2.0.59rc2}/proxynt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proxynt
-Version: 2.0.58
+Version: 2.0.59rc2
 Summary: UNKNOWN
 Home-page: https://github.com/sazima/proxynt
 License: UNKNOWN

{proxynt-2.0.58 → proxynt-2.0.59rc2}/server/tcp_forward_client.py

@@ -111,13 +111,13 @@ class TcpForwardClient:
                 server_socket_list.append(server)
             for c in client_connection_list:
                 c.socket.close()
-                await self.socket_event_loop.async_unregister(c.socket)
+                self.socket_event_loop.unregister(c.socket)
                 self.socket_to_connection.pop(c.socket)
                 self.uid_to_connection.pop(c.uid)
             for s in server_socket_list:
                 self.listen_socket_to_public_server.pop(s.socket_server)
                 try:
-                    await self.socket_event_loop.async_unregister(s.socket_server)
+                    self.socket_event_loop.unregister(s.socket_server)
                     s.socket_server.shutdown(socket.SHUT_RDWR)
                 except OSError:
                     pass
@@ -164,11 +164,11 @@ class TcpForwardClient:
                     LoggerFactory.get_logger().error(f'Close error: {traceback.format_exc()}')
             return
 
-        # --- 发送端限速：在发送前等待 ---
+        # --- 发送端限速：暂停读取，延迟恢复 ---
         if data.speed_limiter and recv:
             wait_time = data.speed_limiter.acquire(len(recv))
             if wait_time > 0:
-                time.sleep(wait_time)
+                self.socket_event_loop.pause_and_resume_later(each, wait_time)
 
         if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
             LoggerFactory.get_logger().debug(f'send to ws uid: {socket_connection.uid}, len: {len(recv)}')
@@ -235,9 +235,8 @@ class TcpForwardClient:
         if uid not in self.uid_to_connection:
             LoggerFactory.get_logger().debug(f'{message}, {uid} not in ')
             return
-        if uid not in self.uid_to_send_lock:
-            self.uid_to_send_lock[uid] = AsyncioLock()
-        async with self.uid_to_send_lock[uid]:
+        lock = self.uid_to_send_lock.setdefault(uid, AsyncioLock())
+        async with lock:
             connection = self.uid_to_connection.get(uid)
             if not connection:
                 return
@@ -273,7 +272,7 @@ class TcpForwardClient:
             connection.socket_server.delete_client(connection)
             # Ensure unregister before closing
             try:
-                await self.socket_event_loop.async_unregister(connection.socket)
+                self.socket_event_loop.unregister(connection.socket)
             except Exception as e:
                 LoggerFactory.get_logger().error(f'Error unregistering socket: {e}')
 
@@ -288,13 +287,13 @@ class TcpForwardClient:
     def close_connection(self, connection: PublicSocketConnection):
         try:
             LoggerFactory.get_logger().info(f'Closing connection {connection.uid}')
-            # with self.close_lock:
             uid = connection.uid
             if uid not in self.uid_to_connection:
                 return
             self.socket_event_loop.unregister(connection.socket)
-            self.uid_to_connection.pop(uid)
-            self.socket_to_connection.pop(connection.socket)
+            self.uid_to_connection.pop(uid, None)
+            self.socket_to_connection.pop(connection.socket, None)
+            self.uid_to_send_lock.pop(uid, None)
             connection.socket_server.delete_client(connection)
             connection.socket.close()
         except Exception:
@@ -306,7 +305,7 @@ class TcpForwardClient:
         s: socket.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         s.bind(('', port))
-        s.listen(5)
+        s.listen(256)
         return s
 
     def close(self):

{proxynt-2.0.58 → proxynt-2.0.59rc2}/server/websocket_handler.py

@@ -4,7 +4,6 @@ import logging
 import time
 import traceback
 from asyncio import Lock
-from concurrent.futures import ThreadPoolExecutor
 from json import JSONDecodeError
 from typing import List, Dict, Set, Tuple
 
@@ -31,7 +30,7 @@ from exceptions.replay_error import ReplayError
 from exceptions.signature_error import SignatureError
 from server.tcp_forward_client import TcpForwardClient
 
-p = ThreadPoolExecutor(max_workers=100)
+MAX_CONCURRENT_MESSAGES = 64
 
 
 class MyWebSocketaHandler(WebSocketHandler):
@@ -63,6 +62,7 @@ class MyWebSocketaHandler(WebSocketHandler):
 
     def open(self, *args: str, **kwargs: str):
         self.lock = Lock()
+        self._msg_sem = asyncio.Semaphore(MAX_CONCURRENT_MESSAGES)
         self.client_name = None
         self.version = None
 
@@ -115,7 +115,14 @@ class MyWebSocketaHandler(WebSocketHandler):
             raise
 
     def on_message(self, m_bytes):
-        asyncio.ensure_future(self.on_message_async(m_bytes))
+        asyncio.ensure_future(self._on_message_with_backpressure(m_bytes))
+
+    async def _on_message_with_backpressure(self, m_bytes):
+        await self._msg_sem.acquire()
+        try:
+            await self.on_message_async(m_bytes)
+        finally:
+            self._msg_sem.release()
 
     async def on_message_async(self, message):
         tcp_forward_client = TcpForwardClient.get_instance()

proxynt-2.0.58/common/pool.py

@@ -1,203 +0,0 @@
-import asyncio
-import logging
-import threading
-import time
-import weakref
-from concurrent.futures import ThreadPoolExecutor
-from selectors import DefaultSelector, EVENT_READ
-
-import socket
-import traceback
-from typing import Dict, List, Set
-
-from common.logger_factory import LoggerFactory
-from common.register_append_data import ResisterAppendData
-from constant.system_constant import SystemConstant
-
-"""
-这里只监听了 socket  的可读状态 
-"""
-max_workers = 99
-
-
-class SelectPool:
-
-    def __init__(self):
-        self.is_running = True
-        self.fileno_to_client: Dict[int, socket.socket] = dict()
-        self.selector = DefaultSelector()
-        self.waiting_register_socket: Set = set()
-
-        self.socket_to_register_lock: Dict[socket.socket, threading.Lock] = dict()
-        self.socket_to_recv_lock: Dict[socket.socket, threading.Lock] = dict()
-        self.executor = ThreadPoolExecutor(max_workers=max_workers)  #
-
-    def stop(self):
-        self.is_running = False
-
-    def clear(self):
-        self.fileno_to_client.clear()
-        self.waiting_register_socket.clear()
-        self.socket_to_register_lock.clear()
-        self.socket_to_recv_lock.clear()
-
-    def register(self, s: socket.socket, data: ResisterAppendData):
-        self.socket_to_register_lock[s] = threading.Lock()
-        self.socket_to_recv_lock[s] = threading.Lock()
-        self.fileno_to_client[s.fileno()] = s
-        self.selector.register(s, EVENT_READ, data)
-
-    def unregister_and_register_delay(self, s: socket.socket, data: ResisterAppendData, delay_time: int):
-        """取消注册, 并在指定秒后注册"""
-
-        def _register_again():
-            try:
-                if s not in self.socket_to_register_lock:
-                    return
-                is_exceed, remain = data.speed_limiter.is_exceed()
-                if is_exceed:
-                    # 再次延迟检测
-                    if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                        LoggerFactory.get_logger().debug('delay register again, maybe next: %.2f seconds "  ' % (remain / data.speed_limiter.max_speed))
-                    threading.Timer(delay_time, _register_again).start()
-                    return
-                with self.socket_to_register_lock[s]:
-                    if s in self.waiting_register_socket:  # 在等待列表中
-                        self.waiting_register_socket.remove(s)
-                        self.register(s, data)
-            except Exception:
-                LoggerFactory.get_logger().error(traceback.format_exc())
-                raise
-
-        if s in self.waiting_register_socket:  # 不在等待列表中
-            return
-        if s not in self.socket_to_register_lock:
-            return
-        with self.socket_to_register_lock[s]:
-            try:
-                self.selector.unregister(s)
-                self.waiting_register_socket.add(s)
-            except OSError:
-                LoggerFactory.get_logger().error(traceback.format_exc())
-            threading.Timer(delay_time, _register_again).start()
-
-    async def async_unregister(self, s: socket.socket):
-        """添加超时机制的异步取消注册"""
-        try:
-            await asyncio.wait_for(
-                asyncio.get_event_loop().run_in_executor(self.executor, self.unregister, s),
-                timeout=5  # 5秒超时
-            )
-        except asyncio.TimeoutError:
-            LoggerFactory.get_logger().error(f"Timeout unregistering socket {s}")
-            # 强制从跟踪字典中移除
-            if s.fileno() in self.fileno_to_client:
-                self.fileno_to_client.pop(s.fileno())
-            if s in self.socket_to_register_lock:
-                self.socket_to_register_lock.pop(s)
-            if s in self.socket_to_recv_lock:
-                self.socket_to_recv_lock.pop(s)
-            if s in self.waiting_register_socket:
-                self.waiting_register_socket.remove(s)
-
-    def unregister(self, s: socket.socket):
-        if s not in self.socket_to_register_lock:
-            LoggerFactory.get_logger().info('not register socket, skip')
-            return
-        with self.socket_to_register_lock[s]:
-            if s in self.waiting_register_socket:
-                self.waiting_register_socket.remove(s)
-            if s.fileno() in self.fileno_to_client:
-                self.fileno_to_client.pop(s.fileno())
-            try:
-                self.selector.unregister(s)
-            except KeyError:
-                # KeyError 代表已经注销
-                pass
-            except ValueError:
-                # ? value error 代表已经注销?
-                pass
-            except OSError:
-                LoggerFactory.get_logger().error(traceback.format_exc())
-            if s in self.socket_to_register_lock:
-                self.socket_to_register_lock.pop(s)
-            if s in self.socket_to_recv_lock:
-                self.socket_to_recv_lock.pop(s)
-
-    def run(self):
-        while True:
-            if not self.is_running:
-                time.sleep(1)
-                continue
-            try:
-                try:
-                    ready = self.selector.select(timeout=SystemConstant.DEFAULT_TIMEOUT)
-                except OSError:
-                    # 监听列表为空的时候, windows会有os error
-                    # LoggerFactory.get_logger().warn
-                    time.sleep(0.5)
-                    continue
-                for key, mask in ready:
-                    fileno = key.fd
-                    client = self.fileno_to_client.get(fileno)
-                    if client is None:
-                        LoggerFactory.get_logger().warn(f'key error, {fileno}, self.fileno_to_client: {self.fileno_to_client}')
-                        continue
-                    data: ResisterAppendData = key.data
-                    # lock = self.socket_to_recv_lock[client]
-                    # if not lock.acquire(blocking=False):
-                    #     if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                    #         LoggerFactory.get_logger().debug(f'lock continue')
-                    #     time.sleep(.005)
-                    #     continue  # 已被其他线程处理，跳过
-                    self.selector.unregister(client)  # register 防止一直就绪状态 耗cpu
-                    self.executor.submit(self._handle_client, client, data)
-            except Exception:
-                LoggerFactory.get_logger().error(traceback.format_exc())
-                time.sleep(1)
-
-    def _handle_client(self, client, data):
-        try:
-            data.callable_(client, data)
-        except Exception:
-            LoggerFactory.get_logger().error(traceback.format_exc())
-        finally:
-            lock = self.socket_to_register_lock.get(client)
-            if lock is not None:
-                if not lock.acquire(blocking=True):  # 正在锁，跳过
-                    return
-                try:
-                    if client not in self.socket_to_register_lock:
-                        LoggerFactory.get_logger().warning('[POOL] re-register skipped: lock removed, fd=%s' % client.fileno())
-                        return
-                    self.selector.register(client, EVENT_READ, data)
-                except Exception:
-                    LoggerFactory.get_logger().error('[POOL] re-register failed fd=%s: %s' % (client.fileno(), traceback.format_exc()))
-                finally:
-                    lock.release()
-            else:
-                if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                    LoggerFactory.get_logger().debug('[POOL] re-register skipped: lock is None, fd=%s' % client.fileno())
-            # self.
-            # if client in self.socket_to_recv_lock:
-            #     self.socket_to_recv_lock[client].release()
-
-        #     try:
-        #         start = time.time()
-        #         if client in self.socket_to_register_lock:
-        #             with self.socket_to_register_lock[client]:
-        #                 try:
-        #                     self.selector.register(client, EVENT_READ, data)
-        #                 except Exception as e:
-        #                     LoggerFactory.get_logger().warning(f'register error {e}')
-        #         else:
-        #             LoggerFactory.get_logger().warning(f'client not in lock')
-        #         if lock.locked():
-        #             lock.release()
-        #         else:
-        #             LoggerFactory.get_logger().warning(f'lock not in lock')
-        #         # if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-        #         #     LoggerFactory.get_logger().debug(f'register cost {time.time() - start} seconds')
-        #     except Exception:
-        #         LoggerFactory.get_logger().error(traceback.format_exc())
-        #