proxynt 2.0.61rc2__tar.gz → 2.1.0__tar.gz

{proxynt-2.0.61rc2 → proxynt-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proxynt
-Version: 2.0.61rc2
+Version: 2.1.0
 Summary: UNKNOWN
 Home-page: https://github.com/sazima/proxynt
 License: UNKNOWN

{proxynt-2.0.61rc2 → proxynt-2.1.0}/client/tcp_forward_client.py

@@ -9,7 +9,7 @@ from typing import Dict, List
 
 from common.logger_factory import LoggerFactory
 from common.nat_serialization import NatSerialization
-from common.pool import SelectPool
+from common.pool import SelectPool, PAUSE_REASON_FLOW_CONTROL
 from common.register_append_data import ResisterAppendData
 from common.speed_limit import SpeedLimiter
 from constant.message_type_constnat import MessageTypeConstant
@@ -38,6 +38,7 @@ class TcpForwardClient:
         self.ws = ws_sender  # WsSender — thread-safe, send via IOLoop.add_callback
         self.lock = Lock()
         self.socket_event_loop = SelectPool()
+        self.flow_control_enabled = False  # 服务端是否支持按连接流控, 由 PUSH_CONFIG 协商
 
         # C2C client-to-client forward state
         self.c2c_rules: List[dict] = []
@@ -382,6 +383,18 @@ class TcpForwardClient:
         )
         LoggerFactory.get_logger().debug('Send to websocket cost time %s' % (time.time() - start_time))
 
+    def set_local_reading_paused(self, uid: bytes, pause: bool):
+        """服务端通知该 uid 积压过多, 暂停/恢复读取本地连接"""
+        connection = self.uid_to_socket_connection.get(uid)
+        if not connection:
+            return
+        if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
+            LoggerFactory.get_logger().debug('flow control %s uid: %s' % ('pause' if pause else 'resume', uid.hex()))
+        if pause:
+            self.socket_event_loop.pause_reading(connection.socket, PAUSE_REASON_FLOW_CONTROL)
+        else:
+            self.socket_event_loop.resume_reading(connection.socket, PAUSE_REASON_FLOW_CONTROL)
+
     def send_by_uid(self, uid: bytes, msg: bytes):
         connection = self.uid_to_socket_connection.get(uid)
         if not connection:

{proxynt-2.0.61rc2 → proxynt-2.1.0}/common/nat_serialization_v1.py

@@ -74,6 +74,8 @@ class NatSerializationV1:
                                    magic, mode_flag, len(target_client), len(target_service), len(source_rule_name), len(protocol),
                                    uid, target_client, target_service, source_rule_name, protocol)
 
+        elif type_ in (MessageTypeConstant.FLOW_PAUSE, MessageTypeConstant.FLOW_RESUME):
+            body = struct.pack(f'{UID_LEN}s', data['data']['uid'])
         elif type_ == MessageTypeConstant.PUSH_CONFIG:
             body = json.dumps(data).encode()
         elif type_ == MessageTypeConstant.PING:
@@ -190,6 +192,13 @@ class NatSerializationV1:
             }
             return return_data
 
+        elif type_.decode() in (MessageTypeConstant.FLOW_PAUSE, MessageTypeConstant.FLOW_RESUME):
+            return_data: MessageEntity = {
+                'type_': type_.decode(),
+                'data': {'uid': body[:UID_LEN]}
+            }
+            return return_data
+
         elif type_ == MessageTypeConstant.PUSH_CONFIG.encode():
             return_data: MessageEntity = json.loads(body.decode())
             return return_data

{proxynt-2.0.61rc2 → proxynt-2.1.0}/common/pool.py

@@ -4,6 +4,7 @@ import threading
 import time
 import traceback
 from selectors import DefaultSelector, EVENT_READ
+from typing import Dict, Set
 
 from common.logger_factory import LoggerFactory
 from common.register_append_data import ResisterAppendData
@@ -14,6 +15,9 @@ from constant.system_constant import SystemConstant
 参考 shadowsocks eventloop 模式
 """
 
+PAUSE_REASON_SPEED_LIMIT = 'speed_limit'
+PAUSE_REASON_FLOW_CONTROL = 'flow_control'
+
 
 class SelectPool:
 
@@ -26,6 +30,9 @@ class SelectPool:
         self._wakeup_r.setblocking(False)
         self._wakeup_w.setblocking(False)
         self.selector.register(self._wakeup_r, EVENT_READ, None)
+        # 每个 socket 的暂停原因 (如 speed_limit / flow_control), 全部解除才恢复读取
+        self._pause_reasons: Dict[socket.socket, Set[str]] = dict()
+        self._pause_lock = threading.Lock()
 
     def _wakeup(self):
         try:
@@ -59,6 +66,8 @@ class SelectPool:
             pass
         if fileno in self.fileno_to_client:
             self.fileno_to_client.pop(fileno)
+        with self._pause_lock:
+            self._pause_reasons.pop(s, None)
         try:
             self.selector.unregister(s)
         except (KeyError, ValueError):
@@ -66,27 +75,43 @@ class SelectPool:
         except OSError:
             LoggerFactory.get_logger().error(traceback.format_exc())
 
-    def pause_reading(self, s: socket.socket):
+    def pause_reading(self, s: socket.socket, reason: str = PAUSE_REASON_SPEED_LIMIT):
         """暂停监听可读事件 (modify 掩码为 0)"""
-        try:
-            key = self.selector.get_key(s)
-            self.selector.modify(s, 0, key.data)
-        except (KeyError, ValueError):
-            pass
+        with self._pause_lock:
+            try:
+                key = self.selector.get_key(s)
+            except (KeyError, ValueError):
+                return
+            reasons = self._pause_reasons.setdefault(s, set())
+            first_pause = not reasons
+            reasons.add(reason)
+            if first_pause:
+                try:
+                    self.selector.modify(s, 0, key.data)
+                except (KeyError, ValueError):
+                    pass
 
-    def resume_reading(self, s: socket.socket):
-        """恢复监听可读事件"""
-        try:
-            key = self.selector.get_key(s)
-            self.selector.modify(s, EVENT_READ, key.data)
-            self._wakeup()
-        except (KeyError, ValueError):
-            pass
+    def resume_reading(self, s: socket.socket, reason: str = PAUSE_REASON_SPEED_LIMIT):
+        """解除一个暂停原因, 没有其他原因时恢复监听可读事件"""
+        with self._pause_lock:
+            reasons = self._pause_reasons.get(s)
+            if not reasons or reason not in reasons:
+                return
+            reasons.discard(reason)
+            if reasons:
+                return
+            self._pause_reasons.pop(s, None)
+            try:
+                key = self.selector.get_key(s)
+                self.selector.modify(s, EVENT_READ, key.data)
+            except (KeyError, ValueError):
+                return
+        self._wakeup()
 
     def pause_and_resume_later(self, s: socket.socket, delay_time: float):
         """暂停可读监听, delay_time 秒后恢复"""
-        self.pause_reading(s)
-        threading.Timer(delay_time, self.resume_reading, args=(s,)).start()
+        self.pause_reading(s, PAUSE_REASON_SPEED_LIMIT)
+        threading.Timer(delay_time, self.resume_reading, args=(s, PAUSE_REASON_SPEED_LIMIT)).start()
 
     def run(self):
         while True:

{proxynt-2.0.61rc2 → proxynt-2.1.0}/constant/message_type_constnat.py

@@ -28,4 +28,9 @@ class MessageTypeConstant:
     P2P_PRE_CONNECT = 'g'
     P2P_EXCHANGE = 'h'        # UDP exchange to establish NAT mapping
     P2P_PEER_INFO = 'i'       # Server sends peer's actual UDP port
-    P2P_PUNCH_REQUEST = 'j'   # Request to initiate P2P hole punching
+    P2P_PUNCH_REQUEST = 'j'   # Request to initiate P2P hole punching
+
+    # 按连接流控: 接收方某个 uid 积压过多时通知发送方暂停/恢复读取该 uid 的数据源
+    # 仅在双方通过 PUSH_CONFIG 协商支持后发送 (旧版本反序列化未知类型会报错)
+    FLOW_PAUSE = 'k'
+    FLOW_RESUME = 'l'

{proxynt-2.0.61rc2 → proxynt-2.1.0}/constant/system_constant.py

@@ -15,7 +15,7 @@ class SystemConstant:
 
     COOKIE_EXPIRE_SECONDS = 3600 * 24
 
-    VERSION = '2.0.61rc2'
+    VERSION = '2.1.0'
 
     GITHUB = 'https://github.com/sazima/proxynt'
 

{proxynt-2.0.61rc2 → proxynt-2.1.0}/proxynt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proxynt
-Version: 2.0.61rc2
+Version: 2.1.0
 Summary: UNKNOWN
 Home-page: https://github.com/sazima/proxynt
 License: UNKNOWN

{proxynt-2.0.61rc2 → proxynt-2.1.0}/run_client.py

@@ -79,6 +79,7 @@ class WsSender:
     """Thread-safe WebSocket sender wrapping Tornado WebSocketClientConnection."""
 
     MAX_PENDING_WRITES = 32
+    WRITE_BACKPRESSURE_TIMEOUT = 10
 
     def __init__(self):
         self._ws = None
@@ -104,24 +105,30 @@ class WsSender:
             io.add_callback(ws.write_message, data, binary=True)
             return
 
-        acquired = self._write_sem.acquire(timeout=10)
-        if not acquired or self._ws is None:
+        write_sem = self._write_sem
+        acquired = write_sem.acquire(timeout=self.WRITE_BACKPRESSURE_TIMEOUT)
+        if self._ws is None:
+            if acquired:
+                write_sem.release()
             return
-        io.add_callback(self._do_write, data)
+        if not acquired:
+            # 不能丢弃: TCP 流中间少一块数据会导致对端协议错乱(如 sshd Bad packet length)
+            LoggerFactory.get_logger().warning(f'websocket write backlog over {self.WRITE_BACKPRESSURE_TIMEOUT}s, send without backpressure')
+        io.add_callback(self._do_write, data, write_sem if acquired else None)
 
-    def _do_write(self, data):
+    def _do_write(self, data, write_sem):
         ws = self._ws
         if ws is None:
-            self._write_sem.release()
+            if write_sem:
+                write_sem.release()
             return
         try:
             future = ws.write_message(data, binary=True)
-            ioloop.IOLoop.current().add_future(future, lambda f: self._on_write_done())
+            if write_sem:
+                ioloop.IOLoop.current().add_future(future, lambda f: write_sem.release())
         except Exception:
-            self._write_sem.release()
-
-    def _on_write_done(self):
-        self._write_sem.release()
+            if write_sem:
+                write_sem.release()
 
     def close(self):
         ws = self._ws
@@ -295,6 +302,7 @@ class WebsocketClient:
                 push_config: PushConfigEntity = message_data['data']
                 self.public_ip = push_config.get('public_ip')
                 self.public_port = push_config.get('public_port')
+                self.forward_client.flow_control_enabled = bool(push_config.get('server_flow_control', False))
 
                 for d in push_config['config_list']:
                     if d.get('speed_limit'):
@@ -306,6 +314,11 @@ class WebsocketClient:
                     self.forward_client.setup_c2c_tcp_listeners(c2c_rules)
                     self.udp_forward_client.setup_c2c_udp_listeners(c2c_rules)
 
+            elif msg_type == MessageTypeConstant.FLOW_PAUSE:
+                self.forward_client.set_local_reading_paused(message_data['data']['uid'], True)
+            elif msg_type == MessageTypeConstant.FLOW_RESUME:
+                self.forward_client.set_local_reading_paused(message_data['data']['uid'], False)
+
             # P2P punch disabled
             elif msg_type == MessageTypeConstant.P2P_PUNCH_REQUEST:
                 LoggerFactory.get_logger().info('P2P punch disabled, ignoring request')
@@ -337,7 +350,8 @@ class WebsocketClient:
                     'config_list': push_client_data,
                     "client_name": client_name,
                     'version': SystemConstant.VERSION,
-                    'p2p_supported': False
+                    'p2p_supported': False,
+                    'flow_control': True
                 }
                 message: MessageEntity = {
                     'type_': MessageTypeConstant.PUSH_CONFIG,

{proxynt-2.0.61rc2 → proxynt-2.1.0}/server/tcp_forward_client.py

@@ -8,14 +8,15 @@ from collections import defaultdict
 from functools import partial
 from threading import Thread, Lock
 from asyncio import Lock as AsyncioLock
-from typing import Dict, Tuple, List, Set
+from collections import deque
+from typing import Dict, Tuple, List, Set, Optional
 import os
 
 import tornado
 
 from common.logger_factory import LoggerFactory
 from common.nat_serialization import NatSerialization
-from common.pool import SelectPool
+from common.pool import SelectPool, PAUSE_REASON_FLOW_CONTROL
 from common.register_append_data import ResisterAppendData
 from common.speed_limit import SpeedLimiter
 from constant.message_type_constnat import MessageTypeConstant
@@ -56,8 +57,8 @@ class PublicSocketConnection:
         # Optimistic send mode configuration
         self.optimistic_mode = True  # Enable optimistic send by default
         self.connection_confirmed = False  # Whether connection is confirmed
-        self.connect_requested = False  # Whether REQUEST_TO_CONNECT has been sent
         self.early_data_buffer = []  # Buffer early data (data received before connection confirmation)
+        self.early_data_lock = Lock()  # socket 线程缓冲与 tornado 线程 flush 互斥, 防止数据乱序/丢失
         self.max_early_data_packets = 65536 * 4  # Maximum buffered packets
         self.bytes_from_public: int = 0
         self.bytes_to_public: int = 0
@@ -71,6 +72,70 @@ class PublicSocketConnection:
         return f'{self.uid}_{self.socket}_{self.socket_server.name}'
 
 
+SEND_QUEUE_HIGH_WATER = 4 * 1024 * 1024  # 单个 uid 积压超过该值时对来源 websocket 做背压
+SEND_QUEUE_LOW_WATER = 1 * 1024 * 1024
+# 客户端支持流控时: 超过 FLOW_HIGH_WATER 通知客户端暂停该 uid, 低于 FLOW_LOW_WATER 恢复
+# 超过 FLOW_HARD_LIMIT 仍退回 websocket 级背压, 防止对端不响应时内存失控
+FLOW_HIGH_WATER = 1 * 1024 * 1024
+FLOW_LOW_WATER = 256 * 1024
+FLOW_HARD_LIMIT = 8 * 1024 * 1024
+
+
+class UidSendQueue:
+    """单个 uid 的有序发送队列, 只在 tornado 线程中使用"""
+
+    def __init__(self):
+        self._items = deque()
+        self._event = asyncio.Event()
+        self._drain_waiters: List[asyncio.Future] = []
+        self.size = 0
+        self.closed = False
+        self.remote_paused = False  # 是否已通知客户端暂停该 uid
+
+    def put(self, message: bytes):
+        if self.closed:
+            return
+        self._items.append(message)
+        self.size += len(message)
+        self._event.set()
+
+    async def get(self) -> Optional[bytes]:
+        """返回下一条数据, 队列关闭时返回 None"""
+        while not self._items:
+            if self.closed:
+                return None
+            self._event.clear()
+            await self._event.wait()
+        if self.closed:
+            return None
+        return self._items.popleft()
+
+    def task_done(self, length: int):
+        self.size -= length
+        if self.size <= SEND_QUEUE_LOW_WATER:
+            self._wake_drain_waiters()
+
+    def wait_drained(self) -> asyncio.Future:
+        future = asyncio.get_event_loop().create_future()
+        self._drain_waiters.append(future)
+        return future
+
+    def close(self):
+        if self.closed:
+            return
+        self.closed = True
+        self._items.clear()
+        self.size = 0
+        self._event.set()
+        self._wake_drain_waiters()
+
+    def _wake_drain_waiters(self):
+        waiters, self._drain_waiters = self._drain_waiters, []
+        for future in waiters:
+            if not future.done():
+                future.set_result(None)
+
+
 class TcpForwardClient:
     _instance = None
 
@@ -83,7 +148,7 @@ class TcpForwardClient:
         self.tornado_loop = tornado_loop
         self.close_lock = Lock()
         self.client_name_to_lock: Dict[str, AsyncioLock] = dict()
-        self.uid_to_send_lock: Dict[bytes, AsyncioLock] = dict()  # 防止data/close并发竞态
+        self.uid_to_send_queue: Dict[bytes, 'UidSendQueue'] = dict()  # 每个 uid 一个有序发送队列
 
     @classmethod
     def get_instance(cls) -> 'TcpForwardClient':
@@ -138,6 +203,8 @@ class TcpForwardClient:
         recv_error = None
         try:
             recv = each.recv(data.read_size)
+        except (BlockingIOError, InterruptedError):
+            return  # 非阻塞 socket 的伪就绪, 等下次可读
         except ConnectionResetError as e:
             recv = b''
             recv_error = e
@@ -153,30 +220,29 @@ class TcpForwardClient:
 
         # Optimistic send mode: buffer data if connection not confirmed
         if socket_connection.optimistic_mode and not socket_connection.connection_confirmed:
-            if not recv:
-                # Client closed connection before confirmation, close immediately
-                LoggerFactory.get_logger().info(f'recv empty before confirmed, close uid: {socket_connection.uid}')
-                try:
-                    self.close_connection(socket_connection)
-                except (OSError, ValueError, KeyError):
-                    LoggerFactory.get_logger().error(f'Close error: {traceback.format_exc()}')
-                return
-            if len(socket_connection.early_data_buffer) < socket_connection.max_early_data_packets:
-                socket_connection.early_data_buffer.append(recv)
-                if not socket_connection.connect_requested:
-                    # 第一块数据到达后再发送 REQUEST_TO_CONNECT，确保缓冲区已有数据
-                    socket_connection.connect_requested = True
-                    Thread(target=self.request_to_connect, args=(socket_connection,)).start()
-                if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                    LoggerFactory.get_logger().debug(f'Buffering early data uid: {socket_connection.uid}, len: {len(recv)}, buffer_size: {len(socket_connection.early_data_buffer)}')
-            else:
-                # Too much buffered data, connection may have failed, close connection
-                LoggerFactory.get_logger().warning(f'Too much early data buffered, closing connection uid: {socket_connection.uid}')
-                try:
-                    self.close_connection(socket_connection)
-                except (OSError, ValueError, KeyError):
-                    LoggerFactory.get_logger().error(f'Close error: {traceback.format_exc()}')
-            return
+            with socket_connection.early_data_lock:
+                # 加锁后再判断一次, 避免与 CONNECT_CONFIRMED 的 flush 竞争
+                if not socket_connection.connection_confirmed:
+                    if not recv:
+                        # Client closed connection before confirmation, close immediately
+                        LoggerFactory.get_logger().info(f'recv empty before confirmed, close uid: {socket_connection.uid}')
+                        try:
+                            self.close_connection(socket_connection)
+                        except (OSError, ValueError, KeyError):
+                            LoggerFactory.get_logger().error(f'Close error: {traceback.format_exc()}')
+                        return
+                    if len(socket_connection.early_data_buffer) < socket_connection.max_early_data_packets:
+                        socket_connection.early_data_buffer.append(recv)
+                        if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
+                            LoggerFactory.get_logger().debug(f'Buffering early data uid: {socket_connection.uid}, len: {len(recv)}, buffer_size: {len(socket_connection.early_data_buffer)}')
+                    else:
+                        # Too much buffered data, connection may have failed, close connection
+                        LoggerFactory.get_logger().warning(f'Too much early data buffered, closing connection uid: {socket_connection.uid}')
+                        try:
+                            self.close_connection(socket_connection)
+                        except (OSError, ValueError, KeyError):
+                            LoggerFactory.get_logger().error(f'Close error: {traceback.format_exc()}')
+                    return
 
         # --- 发送端限速：暂停读取，延迟恢复 ---
         if data.speed_limiter and recv:
@@ -216,6 +282,8 @@ class TcpForwardClient:
             client: socket.socket
             # 启用 TCP_NODELAY 减少延迟（禁用 Nagle 算法）
             client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            # 必须非阻塞: loop.sock_sendall 遇到阻塞 socket 会直接阻塞 send, 一个慢连接会卡住整个事件循环
+            client.setblocking(False)
         except OSError:
             return
         server = self.listen_socket_to_public_server[server_socket]
@@ -225,6 +293,9 @@ class TcpForwardClient:
         client_socket_connection = PublicSocketConnection(uid, client, server)
         self.uid_to_connection[uid] = client_socket_connection
         self.socket_to_connection[client] = client_socket_connection
+        # accept 后立即请求客户端建立连接, 不能等公网端先发数据: MySQL/VNC/FTP/SMTP 等由服务端先发数据的协议会卡住
+        # 早期数据与 CONNECT_CONFIRMED 的竞争由 early_data_lock 保证
+        self.request_to_connect(client_socket_connection)
         append_data = ResisterAppendData(self.handle_message, register_append_data.speed_limiter)
         self.socket_event_loop.register(client, append_data)
 
@@ -244,34 +315,72 @@ class TcpForwardClient:
             partial(handler.write_message, NatSerialization.dumps(send_message, ContextUtils.get_password(), handler.compress_support, handler.protocol_version)), True
         )
 
-    async def send_to_socket(self, uid: bytes, message: bytes):
-        send_start_time = time.time()
-        if uid not in self.uid_to_connection:
-            LoggerFactory.get_logger().debug(f'{message}, {uid} not in ')
-            return
-        lock = self.uid_to_send_lock.setdefault(uid, AsyncioLock())
-        async with lock:
-            connection = self.uid_to_connection.get(uid)
-            if not connection:
-                return
-            socket_client = connection.socket
-            if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                LoggerFactory.get_logger().debug(f'send to socket uid: {uid}, len: {len(message)}')
-            try:
-                await asyncio.wait_for(asyncio.get_event_loop().sock_sendall(socket_client, message), timeout=30)
-            except asyncio.TimeoutError:
-                LoggerFactory.get_logger().warn(f"Socket send timeout for {uid.hex()}, closing connection")
-                asyncio.ensure_future(self.close_connection_async(connection, 'send timeout'))
-                return
-            except OSError as e:
-                LoggerFactory.get_logger().warn(f'{uid.hex()} os error: {e}')
-                asyncio.ensure_future(self.close_connection_async(connection, f'send os error: {e}'))
-                return
-            connection.bytes_to_public += len(message)
-            if not message:
-                asyncio.ensure_future(self.close_connection_async(connection, 'client reported inner close'))
-            if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
-                LoggerFactory.get_logger().debug(f'send to socket cost time {time.time() - send_start_time}')
+    def enqueue_send(self, uid: bytes, message: bytes) -> Optional[asyncio.Future]:
+        """
+        按 uid 入队发送, 立即返回, 保证同一 uid 内的数据顺序, 不同 uid 之间互不阻塞
+        队列积压超过高水位时返回一个 future, 调用方 await 它做背压; 否则返回 None
+        """
+        connection = self.uid_to_connection.get(uid)
+        if not connection:
+            LoggerFactory.get_logger().debug(f'{uid.hex()} not in uid_to_connection, drop {len(message)} bytes')
+            return None
+        send_queue = self.uid_to_send_queue.get(uid)
+        if send_queue is None:
+            send_queue = UidSendQueue()
+            self.uid_to_send_queue[uid] = send_queue
+            asyncio.ensure_future(self._send_loop(connection, send_queue))
+        send_queue.put(message)
+        handler = connection.socket_server.websocket_handler
+        if handler.flow_control_support:
+            if send_queue.size > FLOW_HIGH_WATER and not send_queue.remote_paused:
+                send_queue.remote_paused = True
+                handler.send_flow_control(uid, True)
+            high_water = FLOW_HARD_LIMIT
+        else:
+            high_water = SEND_QUEUE_HIGH_WATER
+        if send_queue.size > high_water:
+            return send_queue.wait_drained()
+        return None
+
+    async def _send_loop(self, connection: PublicSocketConnection, send_queue: 'UidSendQueue'):
+        uid = connection.uid
+        loop = asyncio.get_event_loop()
+        try:
+            while True:
+                message = await send_queue.get()
+                if message is None or self.uid_to_connection.get(uid) is not connection:
+                    return
+                if LoggerFactory.get_logger().isEnabledFor(logging.DEBUG):
+                    LoggerFactory.get_logger().debug(f'send to socket uid: {uid.hex()}, len: {len(message)}')
+                try:
+                    await asyncio.wait_for(loop.sock_sendall(connection.socket, message), timeout=30)
+                except asyncio.TimeoutError:
+                    LoggerFactory.get_logger().warn(f"Socket send timeout for {uid.hex()}, closing connection")
+                    await self.close_connection_async(connection, 'send timeout')
+                    return
+                except OSError as e:
+                    LoggerFactory.get_logger().warn(f'{uid.hex()} os error: {e}')
+                    await self.close_connection_async(connection, f'send os error: {e}')
+                    return
+                connection.bytes_to_public += len(message)
+                send_queue.task_done(len(message))
+                if send_queue.remote_paused and send_queue.size <= FLOW_LOW_WATER:
+                    send_queue.remote_paused = False
+                    connection.socket_server.websocket_handler.send_flow_control(uid, False)
+                if not message:
+                    await self.close_connection_async(connection, 'client reported inner close')
+                    return
+        finally:
+            send_queue.close()
+            if self.uid_to_send_queue.get(uid) is send_queue:
+                self.uid_to_send_queue.pop(uid, None)
+
+    def set_public_reading_paused(self, connection: PublicSocketConnection, pause: bool):
+        """客户端通知暂停/恢复读取公网连接"""
+        if pause:
+            self.socket_event_loop.pause_reading(connection.socket, PAUSE_REASON_FLOW_CONTROL)
+        else:
+            self.socket_event_loop.resume_reading(connection.socket, PAUSE_REASON_FLOW_CONTROL)
 
     async def close_connection_async(self, connection: PublicSocketConnection, reason: str = ''):
         """Asynchronously close connection to avoid blocking in event loop"""
@@ -284,7 +393,9 @@ class TcpForwardClient:
             # Remove from tracking dictionaries
             self.uid_to_connection.pop(uid, None)
             self.socket_to_connection.pop(connection.socket, None)
-            self.uid_to_send_lock.pop(uid, None)
+            send_queue = self.uid_to_send_queue.pop(uid, None)
+            if send_queue:
+                send_queue.close()
             connection.socket_server.delete_client(connection)
             # Ensure unregister before closing
             try:
@@ -309,7 +420,10 @@ class TcpForwardClient:
             self.socket_event_loop.unregister(connection.socket)
             self.uid_to_connection.pop(uid, None)
             self.socket_to_connection.pop(connection.socket, None)
-            self.uid_to_send_lock.pop(uid, None)
+            send_queue = self.uid_to_send_queue.pop(uid, None)
+            if send_queue:
+                # 可能在 socket 线程调用, 交给 tornado 线程关闭队列
+                self.tornado_loop.add_callback(send_queue.close)
             connection.socket_server.delete_client(connection)
             connection.socket.close()
         except Exception: