parallel-ssh 2.13.0rc1__py3-none-any.whl → 2.14.0__py3-none-any.whl

{parallel_ssh-2.13.0rc1.dist-info → parallel_ssh-2.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parallel-ssh
-Version: 2.13.0rc1
+Version: 2.14.0
 Summary: Asynchronous parallel SSH library
 Home-page: https://github.com/ParallelSSH/parallel-ssh
 Author: Panos Kittenis

{parallel_ssh-2.13.0rc1.dist-info → parallel_ssh-2.14.0.dist-info}/RECORD

@@ -9,19 +9,19 @@ pssh/clients/__init__.py,sha256=QzNb1FcbjQyxEPtRbog-ZOidMRu0zA718PkRz21SMmQ,836
 pssh/clients/common.py,sha256=fd_jJj8ezDjwr3peN3LonLff5EYSNkMEre0yjlnRjBU,1375
 pssh/clients/reader.py,sha256=9boQMv9xUBGLY53OybJmWvfa6qsA6mudfdfXk4sF2Oc,3092
 pssh/clients/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pssh/clients/base/parallel.py,sha256=cNirNYL0tGw3usWD7XMm0BWVbT_9PNrUjjEOxHLREhw,25010
-pssh/clients/base/single.py,sha256=9MxJwz6eq8UY2mBksDIyg7YZyRkmO2yAQle4Y2jSJjA,26132
+pssh/clients/base/parallel.py,sha256=GZEYNpg-sOCfKcM-jaAaxdW2Ln4_A_f77qhmCbsdll8,24595
+pssh/clients/base/single.py,sha256=_Txr04S11LADbRekTvpS2xBf4C9UV3rTEZ22tPOTdfo,27013
 pssh/clients/native/__init__.py,sha256=dydX5Fae9U9gRBT8koPTM8H9u4t0rIgD1q9XammFYSs,865
 pssh/clients/native/parallel.py,sha256=AjOJoLdhyyUKWp4Leda08cXC-SsyT16XK4PExiOl3-c,24119
-pssh/clients/native/single.py,sha256=zXbMzYxY7_hkVOS2b2tHujqHq4iFWYgXV4KIL2MVFz8,32118
-pssh/clients/native/tunnel.py,sha256=294UpCYOYlhI9sgoZNvjxr5I199GNOmHevz1lbNVjQI,9309
+pssh/clients/native/single.py,sha256=AQWmq_uBUFhZDkX6Cn0jR21BVd8twcYu_UhwMJYex9U,34388
+pssh/clients/native/tunnel.py,sha256=xMfn75O32E-ZzVTFkA7DeHZOO0MXFSGn4aqGHNIb2lc,10650
 pssh/clients/ssh/__init__.py,sha256=HgYZkVJ5QOgaVFp5snwyLZvQbp0Tbdxj8Gbyic2_Ibo,857
 pssh/clients/ssh/parallel.py,sha256=2X47tgPklubHbb7RjUk75IZjidbbOdJq3VstH3KU6KM,11397
-pssh/clients/ssh/single.py,sha256=oSolU4_D-38rkT1eRqOl33js0CmR727tn9wj9T5qXmI,13617
-parallel_ssh-2.13.0rc1.dist-info/COPYING,sha256=ZA2Q9u5AEkH_YoNNDRsz-DBJ6ZuL_foE7RsKFjXd4-c,18093
-parallel_ssh-2.13.0rc1.dist-info/COPYING.LESSER,sha256=AKibDRiqzUEU3s95Ei24e_Nb3a8rxQ44PJyfTCYzkLI,24486
-parallel_ssh-2.13.0rc1.dist-info/LICENSE,sha256=m4cqigcLitMpxL04D7G_AAD1ZMdQI-yOHmgD8VNkuek,26461
-parallel_ssh-2.13.0rc1.dist-info/METADATA,sha256=EsRUli7LAx_verG8ezBIb2hGb1uV_e0ltHIgUMJ9cqE,10733
-parallel_ssh-2.13.0rc1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-parallel_ssh-2.13.0rc1.dist-info/top_level.txt,sha256=s8P6ZHOwt2BYgDc62Cpd2z7i-rebGzIhhnO09pger0U,5
-parallel_ssh-2.13.0rc1.dist-info/RECORD,,
+pssh/clients/ssh/single.py,sha256=z9_OQKYyxOMnTixycZLuBoU2GjQYO_I-VL_L3GddwKo,14068
+parallel_ssh-2.14.0.dist-info/COPYING,sha256=ZA2Q9u5AEkH_YoNNDRsz-DBJ6ZuL_foE7RsKFjXd4-c,18093
+parallel_ssh-2.14.0.dist-info/COPYING.LESSER,sha256=AKibDRiqzUEU3s95Ei24e_Nb3a8rxQ44PJyfTCYzkLI,24486
+parallel_ssh-2.14.0.dist-info/LICENSE,sha256=m4cqigcLitMpxL04D7G_AAD1ZMdQI-yOHmgD8VNkuek,26461
+parallel_ssh-2.14.0.dist-info/METADATA,sha256=Y2Vjl44EdOIuyRpZx2Feav-OLD_6yiiB3u6ly5LgVGc,10730
+parallel_ssh-2.14.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+parallel_ssh-2.14.0.dist-info/top_level.txt,sha256=s8P6ZHOwt2BYgDc62Cpd2z7i-rebGzIhhnO09pger0U,5
+parallel_ssh-2.14.0.dist-info/RECORD,,

pssh/clients/base/parallel.py

@@ -114,20 +114,6 @@ class BaseParallelSSHClient(object):
             self._host_clients.pop((i, host), None)
         self._hosts = _hosts
 
-    def __del__(self):
-        self.disconnect()
-
-    def disconnect(self):
-        """Disconnect all clients."""
-        if not hasattr(self, '_host_clients'):
-            return
-        for s_client in self._host_clients.values():
-            try:
-                s_client.disconnect()
-            except Exception as ex:
-                logger.debug("Client disconnect failed with %s", ex)
-                pass
-
     def _check_host_config(self):
         if self.host_config is None:
             return

pssh/clients/base/single.py

@@ -19,10 +19,13 @@ import logging
 import os
 from getpass import getuser
 from socket import gaierror as sock_gaierror, error as sock_error
+from warnings import warn
 
 from gevent import sleep, socket, Timeout as GTimeout
 from gevent.hub import Hub
 from gevent.select import poll, POLLIN, POLLOUT
+from gevent.socket import SHUT_RDWR
+from gevent.pool import Pool
 from ssh2.exceptions import AgentConnectionError, AgentListIdentitiesError, \
     AgentAuthenticationError, AgentGetIdentityError
 from ssh2.utils import find_eol
@@ -39,6 +42,61 @@ host_logger = logging.getLogger('pssh.host_logger')
 logger = logging.getLogger(__name__)
 
 
+class PollMixIn(object):
+    """MixIn for co-operative socket polling functionality.
+
+    """
+    __slots__ = ('sock',)
+
+    def __init__(self, sock=None):
+        self.sock = sock
+
+    def poll(self, timeout=None):
+        raise NotImplementedError
+
+    def eagain(self, func, *args, **kwargs):
+        raise NotImplementedError
+
+    def eagain_write(self, write_func, data):
+        raise NotImplementedError
+
+    def _poll_errcodes(self, directions_func, inbound, outbound):
+        directions = directions_func()
+        if directions == 0:
+            return
+        events = 0
+        if directions & inbound:
+            events = POLLIN
+        if directions & outbound:
+            events |= POLLOUT
+        self._poll_socket(events)
+
+    def _poll_socket(self, events):
+        if self.sock is None:
+            return
+        poller = poll()
+        poller.register(self.sock, eventmask=events)
+        poller.poll(timeout=1)
+
+    def _eagain_errcode(self, func, eagain, *args, **kwargs):
+        ret = func(*args, **kwargs)
+        while ret == eagain:
+            self.poll()
+            ret = func(*args, **kwargs)
+            sleep()
+        return ret
+
+    def _eagain_write_errcode(self, write_func, data, eagain):
+        data_len = len(data)
+        total_written = 0
+        while total_written < data_len:
+            rc, bytes_written = write_func(data[total_written:])
+            total_written += bytes_written
+            if rc == eagain:
+                self.poll()
+            sleep()
+
+
 class Stdin(object):
     """Stdin stream for a channel.
 
@@ -64,13 +122,13 @@ class Stdin(object):
         :param data: Data to write.
         :type data: str
         """
-        return self._client._eagain(self._channel.write, data)
+        return self._client.eagain(self._channel.write, data)
 
     def flush(self):
         """Flush pending data written to stdin."""
         if not hasattr(self._channel, "flush"):
             return
-        return self._client._eagain(self._channel.flush)
+        return self._client.eagain(self._channel.flush)
 
 
 class InteractiveShell(object):
@@ -132,7 +190,7 @@ class InteractiveShell(object):
         """Wait for shell to finish executing and close channel."""
         if self._chan is None:
             return
-        self._client._eagain(self._chan.send_eof)
+        self._client.eagain(self._chan.send_eof)
         self._client.wait_finished(self.output)
         return self
 
@@ -144,10 +202,10 @@ class InteractiveShell(object):
         :type cmd: str
         """
         cmd = cmd.encode(self._encoding) + self._EOL
-        self._client._eagain_write(self._chan.write, cmd)
+        self._client.eagain_write(self._chan.write, cmd)
 
 
-class BaseSSHClient(object):
+class BaseSSHClient(PollMixIn):
 
     IDENTITIES = (
         os.path.expanduser('~/.ssh/id_rsa'),
@@ -169,6 +227,7 @@ class BaseSSHClient(object):
                  identity_auth=True,
                  ipv6_only=False,
                  ):
+        super(PollMixIn, self).__init__()
         self._auth_thread_pool = _auth_thread_pool
         self.host = host
         self.alias = alias
@@ -176,7 +235,6 @@ class BaseSSHClient(object):
         self.password = password
         self.port = port if port else 22
         self.num_retries = num_retries
-        self.sock = None
         self.timeout = timeout if timeout else None
         self.retry_delay = retry_delay
         self.allow_agent = allow_agent
@@ -187,6 +245,7 @@ class BaseSSHClient(object):
         self.identity_auth = identity_auth
         self._keepalive_greenlet = None
         self.ipv6_only = ipv6_only
+        self._pool = Pool()
         self._init()
 
     def _pkey_from_memory(self, pkey_data):
@@ -212,11 +271,15 @@ class BaseSSHClient(object):
             raise AuthenticationError(msg, self.host, self.port, ex, retries, self.num_retries)
 
     def disconnect(self):
+        """Deprecated and a no-op. Disconnections handled by client de-allocation."""
+        warn("Deprecated and a no-op - to be removed in future releases.", DeprecationWarning)
+
+    def _disconnect(self):
         raise NotImplementedError
 
     def __del__(self):
         try:
-            self.disconnect()
+            self._disconnect()
         except Exception:
             pass
 
@@ -224,7 +287,7 @@ class BaseSSHClient(object):
         return self
 
     def __exit__(self, *args):
-        self.disconnect()
+        self._disconnect()
 
     def open_shell(self, encoding='utf-8', read_timeout=None):
         """Open interactive shell on new channel.
@@ -244,7 +307,8 @@ class BaseSSHClient(object):
         raise NotImplementedError
 
     def _disconnect_eagain(self):
-        self._eagain(self.session.disconnect)
+        if self.session is not None and self.sock is not None and not self.sock.closed:
+            self.eagain(self.session.disconnect)
 
     def _connect_init_session_retry(self, retries):
         try:
@@ -254,9 +318,11 @@ class BaseSSHClient(object):
         self.session = None
         if not self.sock.closed:
             try:
-                self.sock.close()
+                self.sock.shutdown(SHUT_RDWR)
+                self.sock.detach()
             except Exception:
                 pass
+        self.sock = None
         sleep(self.retry_delay)
         self._connect(self._host, self._port, retries=retries)
         return self._init_session(retries=retries)
@@ -420,6 +486,13 @@ class BaseSSHClient(object):
         raise NotImplementedError
 
     def execute(self, cmd, use_pty=False, channel=None):
+        """
+        Deprecated - use ``run_command`` instead which returns a ``HostOutput`` object.
+        """
+        warn("Deprecated - use run_command instead.", DeprecationWarning)
+        return self._execute(cmd, use_pty=use_pty, channel=channel)
+
+    def _execute(self, cmd, use_pty=False, channel=None):
         raise NotImplementedError
 
     def read_stderr(self, stderr_buffer, timeout=None):
@@ -554,37 +627,11 @@ class BaseSSHClient(object):
             _command += "%s '%s'" % (_shell, command,)
         _command = _command.encode(encoding)
         with GTimeout(seconds=self.timeout):
-            channel = self.execute(_command, use_pty=use_pty)
+            channel = self._execute(_command, use_pty=use_pty)
         _timeout = read_timeout if read_timeout else timeout
         host_out = self._make_host_output(channel, encoding, _timeout)
         return host_out
 
-    def _eagain_write_errcode(self, write_func, data, eagain):
-        data_len = len(data)
-        total_written = 0
-        while total_written < data_len:
-            rc, bytes_written = write_func(data[total_written:])
-            total_written += bytes_written
-            if rc == eagain:
-                self.poll()
-            sleep()
-
-    def _eagain_errcode(self, func, eagain, *args, **kwargs):
-        timeout = kwargs.pop('timeout', self.timeout)
-        with GTimeout(seconds=timeout, exception=Timeout):
-            ret = func(*args, **kwargs)
-            while ret == eagain:
-                self.poll()
-                ret = func(*args, **kwargs)
-                sleep()
-            return ret
-
-    def _eagain_write(self, write_func, data):
-        raise NotImplementedError
-
-    def _eagain(self, func, *args, **kwargs):
-        raise NotImplementedError
-
     def _make_sftp(self):
         raise NotImplementedError
 
@@ -692,24 +739,3 @@ class BaseSSHClient(object):
         _sep = file_path.rfind('/')
         if _sep > 0:
             return file_path[:_sep]
-
-    def poll(self):
-        raise NotImplementedError
-
-    def _poll_socket(self, events):
-        if self.sock is None:
-            return
-        poller = poll()
-        poller.register(self.sock, eventmask=events)
-        poller.poll(timeout=1)
-
-    def _poll_errcodes(self, directions_func, inbound, outbound):
-        directions = directions_func()
-        if directions == 0:
-            return
-        events = 0
-        if directions & inbound:
-            events = POLLIN
-        if directions & outbound:
-            events |= POLLOUT
-        self._poll_socket(events)

pssh/clients/native/single.py

@@ -15,12 +15,15 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-import logging
-import os
 from collections import deque
 
-from gevent import sleep, spawn, get_hub
+import logging
+import os
+from gevent import sleep, get_hub
 from gevent.lock import RLock
+from gevent.pool import Pool
+from gevent.socket import SHUT_RDWR
+from gevent.timeout import Timeout as GTimeout
 from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN
 from ssh2.exceptions import SFTPHandleError, SFTPProtocolError, \
     Timeout as SSH2Timeout
@@ -31,7 +34,7 @@ from ssh2.sftp import LIBSSH2_FXF_READ, LIBSSH2_FXF_CREAT, LIBSSH2_FXF_WRITE, \
     LIBSSH2_SFTP_S_IXGRP, LIBSSH2_SFTP_S_IXOTH
 
 from .tunnel import FORWARDER
-from ..base.single import BaseSSHClient
+from ..base.single import BaseSSHClient, PollMixIn
 from ...constants import DEFAULT_RETRIES, RETRY_DELAY
 from ...exceptions import SessionError, SFTPError, \
     SFTPIOError, Timeout, SCPError, ProxyError
@@ -41,6 +44,51 @@ logger = logging.getLogger(__name__)
 THREAD_POOL = get_hub().threadpool
 
 
+class KeepAlive(PollMixIn):
+    """Class for handling SSHClient keepalive functionality.
+
+    Spawns a greenlet in its own pool for sending keepalives to a given session.
+    """
+    __slots__ = ('sock', 'session', '_let', '_pool')
+
+    def __init__(self, sock, session):
+        """
+        :param sock: The socket session is using to communicate.
+        :type sock: :py:class:`gevent.socket.socket`
+        :param session: The session keepalive is configured on.
+        :type session: :py:class:`ssh2.session.Session`
+        """
+        super(PollMixIn, self).__init__()
+        self._pool = Pool(1)
+        self.sock = sock
+        self.session = session
+        self._let = self._pool.spawn(self._send_keepalive)
+        self._let.start()
+
+    def _send_keepalive(self):
+        while True:
+            if self.session is None or self.sock is None or self.sock.closed:
+                return
+            sleep_for = self.eagain(self.session.keepalive_send)
+            sleep(sleep_for)
+
+    def poll(self, timeout=None):
+        """Perform co-operative gevent poll on ssh2 session socket.
+
+        Blocks current greenlet only if socket has pending read or write operations
+        in the appropriate direction.
+        :param timeout: Deprecated and unused - to be removed.
+        """
+        self._poll_errcodes(
+            self.session.block_directions,
+            LIBSSH2_SESSION_BLOCK_INBOUND,
+            LIBSSH2_SESSION_BLOCK_OUTBOUND,
+        )
+
+    def eagain(self, func, *args, **kwargs):
+        return self._eagain_errcode(func, LIBSSH2_ERROR_EAGAIN, *args, **kwargs)
+
+
 class SSHClient(BaseSSHClient):
     """ssh2-python (libssh2) based non-blocking SSH client."""
     # 2MB buffer
@@ -58,7 +106,8 @@ class SSHClient(BaseSSHClient):
                  proxy_pkey=None,
                  proxy_user=None,
                  proxy_password=None,
-                 _auth_thread_pool=True, keepalive_seconds=60,
+                 _auth_thread_pool=True,
+                 keepalive_seconds=60,
                  identity_auth=True,
                  ipv6_only=False,
                  ):
@@ -85,7 +134,9 @@ class SSHClient(BaseSSHClient):
           to :py:class:`pssh.constants.RETRY_DELAY`
         :type retry_delay: int or float
         :param timeout: SSH session timeout setting in seconds. This controls
-          timeout setting of authenticated SSH sessions.
+          timeout setting of authenticated SSH sessions for each individual SSH operation.
+          Also currently sets socket as well as per function timeout in some cases, see
+          function descriptions.
         :type timeout: int or float
         :param allow_agent: (Optional) set to False to disable connecting to
           the system's SSH agent
@@ -146,7 +197,7 @@ class SSHClient(BaseSSHClient):
         )
 
     def _shell(self, channel):
-        return self._eagain(channel.shell)
+        return self.eagain(channel.shell)
 
     def _connect_proxy(self, proxy_host, proxy_port, proxy_pkey,
                        user=None, password=None, alias=None,
@@ -178,42 +229,36 @@ class SSHClient(BaseSSHClient):
         proxy_local_port = FORWARDER.out_q.get()
         return proxy_local_port
 
-    def disconnect(self):
+    def _disconnect(self):
         """Attempt to disconnect session.
 
         Any errors on calling disconnect are suppressed by this function.
+
+        Does not need to be called directly - called when client object is de-allocated.
         """
         self._keepalive_greenlet = None
-        if self.session is not None:
+        if self.session is not None and self.sock is not None and not self.sock.closed:
             try:
                 self._disconnect_eagain()
             except Exception:
                 pass
-            self.session = None
+        self.session = None
+        # To allow for file descriptor reuse, which is part of gevent, shutdown but do not close socket here.
+        # Done by gevent when file descriptor is closed.
+        if self.sock is not None and not self.sock.closed:
+            try:
+                self.sock.shutdown(SHUT_RDWR)
+                self.sock.detach()
+            except Exception:
+                pass
+        self.sock = None
+        # Notify forwarder that proxy tunnel server can be shutdown
         if isinstance(self._proxy_client, SSHClient):
-            # Don't disconnect proxy client here - let the TunnelServer do it at the time that
-            # _wait_send_receive_lets ends. The cleanup_server call here triggers the TunnelServer
-            # to stop.
             FORWARDER.cleanup_server(self._proxy_client)
 
-            # I wanted to clean up all the sockets here to avoid a ResourceWarning from unittest,
-            # but unfortunately closing this socket here causes a segfault, not sure why yet.
-            # self.sock.close()
-        else:
-            self.sock.close()
-        self.sock = None
-
-    def spawn_send_keepalive(self):
-        """Spawns a new greenlet that sends keep alive messages every
-        self.keepalive_seconds"""
-        return spawn(self._send_keepalive)
-
-    def _send_keepalive(self):
-        while True:
-            sleep(self._eagain(self.session.keepalive_send))
-
     def configure_keepalive(self):
         """Configures keepalive on the server for `self.keepalive_seconds`."""
+        # Configure keepalives without a reply.
         self.session.keepalive_config(False, self.keepalive_seconds)
 
     def _init_session(self, retries=1):
@@ -234,7 +279,11 @@ class SSHClient(BaseSSHClient):
             msg = "Error connecting to host %s:%s - %s"
             logger.error(msg, self.host, self.port, ex)
             if not self.sock.closed:
-                self.sock.close()
+                try:
+                    self.sock.shutdown(SHUT_RDWR)
+                    self.sock.detach()
+                except Exception:
+                    pass
             if isinstance(ex, SSH2Timeout):
                 raise Timeout(msg, self.host, self.port, ex)
             raise
@@ -242,7 +291,7 @@ class SSHClient(BaseSSHClient):
     def _keepalive(self):
         if self.keepalive_seconds:
             self.configure_keepalive()
-            self._keepalive_greenlet = self.spawn_send_keepalive()
+            self._keepalive_greenlet = KeepAlive(self.sock, self.session)
 
     def _agent_auth(self):
         self.session.agent_auth(self.user)
@@ -264,7 +313,7 @@ class SSHClient(BaseSSHClient):
         self.session.userauth_password(self.user, self.password)
 
     def _open_session(self):
-        chan = self._eagain(self.session.open_session)
+        chan = self.eagain(self.session.open_session)
         return chan
 
     def open_session(self):
@@ -282,14 +331,18 @@ class SSHClient(BaseSSHClient):
         return chan
 
     def _make_output_readers(self, channel, stdout_buffer, stderr_buffer):
-        _stdout_reader = spawn(
+        # TODO: These greenlets need to be outside client scope or we create a reader <-> client cyclical reference
+        _stdout_reader = self._pool.spawn(
             self._read_output_to_buffer, channel.read, stdout_buffer)
-        _stderr_reader = spawn(
+        _stderr_reader = self._pool.spawn(
             self._read_output_to_buffer, channel.read_stderr, stderr_buffer)
         return _stdout_reader, _stderr_reader
 
-    def execute(self, cmd, use_pty=False, channel=None):
-        """Execute command on remote server.
+    def _execute(self, cmd, use_pty=False, channel=None):
+        """
+        Use ``run_command`` which returns a ``HostOutput`` object rather than this function directly.
+
+        Execute command on remote server.
 
         :param cmd: Command to execute.
         :type cmd: str
@@ -298,12 +351,14 @@ class SSHClient(BaseSSHClient):
         :param channel: Use provided channel for execute rather than creating
           a new one.
         :type channel: :py:class:`ssh2.channel.Channel`
+
+        :rtype: :py:class:`ssh2.channel.Channel`
         """
         channel = self.open_session() if channel is None else channel
         if use_pty:
-            self._eagain(channel.pty)
+            self.eagain(channel.pty)
         logger.debug("Executing command '%s'", cmd)
-        self._eagain(channel.execute, cmd)
+        self.eagain(channel.execute, cmd)
         return channel
 
     def _read_output_to_buffer(self, read_func, _buffer, is_stderr=False):
@@ -341,20 +396,23 @@ class SSHClient(BaseSSHClient):
         channel = host_output.channel
         if channel is None:
             return
-        self._eagain(channel.wait_eof, timeout=timeout)
-        # Close channel to indicate no more commands will be sent over it
-        self.close_channel(channel)
+        with GTimeout(seconds=timeout, exception=Timeout):
+            self.eagain(channel.wait_eof)
+            # Close channel to indicate no more commands will be sent over it
+            self.close_channel(channel)
 
     def close_channel(self, channel):
+        """Close given channel, handling EAGAIN."""
         with self._chan_stdout_lock, self._chan_stderr_lock:
             logger.debug("Closing channel")
-            self._eagain(channel.close)
+            self.eagain(channel.close)
 
-    def _eagain(self, func, *args, **kwargs):
+    def eagain(self, func, *args, **kwargs):
+        """Handle EAGAIN and call given function with any args, polling for as long as there is data to receive."""
         return self._eagain_errcode(func, LIBSSH2_ERROR_EAGAIN, *args, **kwargs)
 
     def _make_sftp_eagain(self):
-        return self._eagain(self.session.sftp_init)
+        return self.eagain(self.session.sftp_init)
 
     def _make_sftp(self):
         try:
@@ -381,7 +439,7 @@ class SSHClient(BaseSSHClient):
             LIBSSH2_SFTP_S_IXGRP | \
             LIBSSH2_SFTP_S_IXOTH
         try:
-            self._eagain(sftp.mkdir, directory, mode)
+            self.eagain(sftp.mkdir, directory, mode)
         except SFTPProtocolError as error:
             msg = "Error occured creating directory %s on host %s - %s"
             logger.error(msg, directory, self.host, error)
@@ -419,7 +477,7 @@ class SSHClient(BaseSSHClient):
         destination = self._remote_paths_split(remote_file)
         if destination is not None:
             try:
-                self._eagain(sftp.stat, destination)
+                self.eagain(sftp.stat, destination)
             except (SFTPHandleError, SFTPProtocolError):
                 self.mkdir(sftp, destination)
         self.sftp_put(sftp, local_file, remote_file)
@@ -482,7 +540,7 @@ class SSHClient(BaseSSHClient):
             cur_dir = _paths_to_create.popleft()
             cwd = '/'.join([cwd, cur_dir])
             try:
-                self._eagain(sftp.stat, cwd)
+                self.eagain(sftp.stat, cwd)
             except (SFTPHandleError, SFTPProtocolError) as ex:
                 logger.debug("Stat for %s failed with %s", cwd, ex)
                 self._mkdir(sftp, cwd)
@@ -514,7 +572,7 @@ class SSHClient(BaseSSHClient):
         """
         sftp = self._make_sftp() if sftp is None else sftp
         try:
-            self._eagain(sftp.stat, remote_file)
+            self.eagain(sftp.stat, remote_file)
         except (SFTPHandleError, SFTPProtocolError):
             msg = "Remote file or directory %s on host %s does not exist"
             logger.error(msg, remote_file, self.host)
@@ -541,7 +599,7 @@ class SSHClient(BaseSSHClient):
 
     def _scp_recv_recursive(self, remote_file, local_file, sftp, encoding='utf-8'):
         try:
-            self._eagain(sftp.stat, remote_file)
+            self.eagain(sftp.stat, remote_file)
         except (SFTPHandleError, SFTPProtocolError):
             msg = "Remote file or directory %s does not exist"
             logger.error(msg, remote_file)
@@ -602,7 +660,7 @@ class SSHClient(BaseSSHClient):
 
     def _scp_recv(self, remote_file, local_file):
         try:
-            (file_chan, fileinfo) = self._eagain(
+            (file_chan, fileinfo) = self.eagain(
                 self.session.scp_recv2, remote_file)
         except Exception as ex:
             msg = "Error copying file %s from host %s - %s"
@@ -661,7 +719,7 @@ class SSHClient(BaseSSHClient):
             if destination is not None:
                 sftp = self._make_sftp() if sftp is None else sftp
                 try:
-                    self._eagain(sftp.stat, destination)
+                    self.eagain(sftp.stat, destination)
                 except (SFTPHandleError, SFTPProtocolError):
                     self.mkdir(sftp, destination)
         elif remote_file.endswith('/'):
@@ -674,7 +732,7 @@ class SSHClient(BaseSSHClient):
     def _scp_send(self, local_file, remote_file):
         fileinfo = os.stat(local_file)
         try:
-            chan = self._eagain(
+            chan = self.eagain(
                 self.session.scp_send64,
                 remote_file, fileinfo.st_mode & 0o777, fileinfo.st_size,
                 fileinfo.st_mtime, fileinfo.st_atime)
@@ -693,14 +751,14 @@ class SSHClient(BaseSSHClient):
             logger.error(msg, remote_file, self.host, ex)
             raise SCPError(msg, remote_file, self.host, ex)
         finally:
-            self._eagain(chan.flush)
-            self._eagain(chan.send_eof)
-            self._eagain(chan.wait_eof)
-            self._eagain(chan.wait_closed)
+            self.eagain(chan.flush)
+            self.eagain(chan.send_eof)
+            self.eagain(chan.wait_eof)
+            self.eagain(chan.wait_closed)
 
     def _sftp_openfh(self, open_func, remote_file, *args):
         try:
-            fh = self._eagain(open_func, remote_file, *args)
+            fh = self.eagain(open_func, remote_file, *args)
         except Exception as ex:
             raise SFTPError(ex)
         return fh
@@ -758,12 +816,9 @@ class SSHClient(BaseSSHClient):
             LIBSSH2_SESSION_BLOCK_OUTBOUND,
         )
 
-    def _eagain_write(self, write_func, data):
+    def eagain_write(self, write_func, data):
         """Write data with given write_func for an ssh2-python session while
         handling EAGAIN and resuming writes from last written byte on each call to
         write_func.
         """
         return self._eagain_write_errcode(write_func, data, LIBSSH2_ERROR_EAGAIN)
-
-    def eagain_write(self, write_func, data):
-        return self._eagain_write(write_func, data)

pssh/clients/native/tunnel.py

@@ -15,13 +15,14 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+import atexit
 import logging
-from queue import Queue
-from threading import Thread, Event
-
-from gevent import spawn, joinall, get_hub, sleep
+from gevent import joinall, get_hub, sleep
+from gevent.pool import Pool
 from gevent.server import StreamServer
+from queue import Queue
 from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN
+from threading import Thread, Event
 
 from ...constants import DEFAULT_RETRIES
 
@@ -39,25 +40,39 @@ class LocalForwarder(Thread):
         and get port to connect to via ``out_q`` once a target has been put into the input queue.
 
         ``SSHClient`` is the client for the SSH host that will be proxying.
+
+        There can be as many LocalForwarder(s) as needed to scale.
+
+        Relationship of clients to forwarders to servers is:
+
+        One client -> One Forwarder -> Many Servers <-> Many proxy hosts -> one target host per proxy host
+
+        A Forwarder starts servers. Servers communicate with clients directly via Forwarder thread.
+
+        Multiple forwarder threads can be used to scale clients to more threads as number of clients increases causing
+        contention in forwarder threads handling proxy connections.
         """
-        Thread.__init__(self)
+        Thread.__init__(self, daemon=True)
         self.in_q = Queue(1)
         self.out_q = Queue(1)
+        self._pool = None
         self._servers = {}
         self._hub = None
         self.started = Event()
         self._cleanup_let = None
+        self.shutdown_triggered = Event()
 
     def _start_server(self):
         client, host, port = self.in_q.get()
+        logger.debug("Starting server for %s:%s", host, port)
         server = TunnelServer(client, host, port)
+        self._servers[client] = server
         server.start()
-        self._get_server_listen_port(client, server)
+        self._get_server_listen_port(server)
 
-    def _get_server_listen_port(self, client, server):
+    def _get_server_listen_port(self, server):
         while not server.started:
             sleep(0.01)
-        self._servers[client] = server
         local_port = server.listen_port
         self.out_q.put(local_port)
 
@@ -74,9 +89,20 @@ class LocalForwarder(Thread):
         self.in_q.put((client, host, port))
 
     def shutdown(self):
-        """Stop all tunnel servers."""
+        """Stop all tunnel servers and shutdown LocalForwarder thread.
+
+        This function will join the current thread and wait for it to shutdown if needed.
+        """
         for client, server in self._servers.items():
             server.stop()
+        self._servers = {}
+        if self.started:
+            self.shutdown_triggered.set()
+            try:
+                self.join()
+            except RuntimeError:
+                # Re-entry protection
+                pass
 
     def _cleanup_servers_let(self):
         while True:
@@ -89,19 +115,22 @@ class LocalForwarder(Thread):
                 self.cleanup_server(client)
 
     def run(self):
-        """Thread runner ensures a non main hub has been created for all subsequent
+        """Thread runner ensures a non-main hub has been created for all subsequent
         greenlets and waits for (client, host, port) tuples to be put into self.in_q.
 
-        A server is created once something is in the queue and the port to connect to
-        is put into self.out_q.
+        A server is created once something is in the queue and only then is the port to connect to
+        put into self.out_q.
         """
         self._hub = get_hub()
         assert self._hub.main_hub is False
+        self._pool = Pool(1)
         self.started.set()
-        self._cleanup_let = spawn(self._cleanup_servers_let)
+        self._cleanup_let = self._pool.spawn(self._cleanup_servers_let)
         logger.debug("Hub in server runner is main hub: %s", self._hub.main_hub)
         try:
             while True:
+                if self.shutdown_triggered.is_set():
+                    return
                 if self.in_q.empty():
                     sleep(.01)
                     continue
@@ -111,8 +140,9 @@ class LocalForwarder(Thread):
             self.shutdown()
 
     def cleanup_server(self, client):
-        """The purpose of this function is for a proxied client to notify the LocalForwarder that it
-         is shutting down and its corresponding server can also be shut down."""
+        """
+        Stop server for given proxy client and remove client from this forwarder.
+        """
         server = self._servers[client]
         server.stop()
         del self._servers[client]
@@ -123,16 +153,18 @@ class TunnelServer(StreamServer):
 
     Accepts connections on an available bind_address port once started and tunnels data
     to/from remote SSH host for each connection.
+
+    TunnelServer.listen_port will return listening port for server on given host once TunnelServer.started is True.
     """
 
     def __init__(self, client, host, port, bind_address='127.0.0.1',
                  num_retries=DEFAULT_RETRIES):
-        StreamServer.__init__(self, (bind_address, 0), self._read_rw)
         self.client = client
+        self._pool = Pool()
+        StreamServer.__init__(self, (bind_address, 0), self._read_rw, spawn=self._pool)
         self.host = host
         self.port = port
         self.session = client.session
-        self._client = client
         self._retries = num_retries
         self.bind_address = bind_address
         self.exception = None
@@ -154,41 +186,40 @@ class TunnelServer(StreamServer):
                          self.host, self.port, ex)
             self.exception = ex
             return
-        source = spawn(self._read_forward_sock, socket, channel)
-        dest = spawn(self._read_channel, socket, channel)
+        # Channel remains alive while this handler function is alive
+        source = self._pool.spawn(self._read_forward_sock, socket, channel)
+        dest = self._pool.spawn(self._read_channel, socket, channel)
         logger.debug("Waiting for read/write greenlets")
-        self._source_let = source
-        self._dest_let = dest
         self._wait_send_receive_lets(source, dest, channel)
 
     def _wait_send_receive_lets(self, source, dest, channel):
         try:
             joinall((source, dest), raise_error=True)
         finally:
-            # Forward socket does not need to be closed here; StreamServer does it in do_close
-            logger.debug("Closing channel")
-            self._client.close_channel(channel)
-
-            # Disconnect client here to make sure it happens AFTER close_channel
-            self._client.disconnect()
+            logger.debug("Read/Write greenlets for tunnel target %s:%s finished, closing forwarding channel",
+                         self.host, self.port)
+            self.client.close_channel(channel)
 
     def _read_forward_sock(self, forward_sock, channel):
         while True:
             if channel is None or channel.eof():
                 logger.debug("Channel closed, tunnel forward socket reader exiting")
                 return
+            if not forward_sock or forward_sock.closed:
+                logger.debug("Forward socket closed, tunnel forward socket reader exiting")
+                return
             try:
                 data = forward_sock.recv(1024)
             except Exception as ex:
                 logger.error("Forward socket read error: %s", ex)
                 raise
             data_len = len(data)
-            # logger.debug("Read %s data from forward socket", data_len,)
+            # logger.debug("Read %s data from forward socket", data_len)
             if data_len == 0:
                 sleep(.01)
                 continue
             try:
-                self._client.eagain_write(channel.write, data)
+                self.client.eagain_write(channel.write, data)
             except Exception as ex:
                 logger.error("Error writing data to channel - %s", ex)
                 raise
@@ -204,9 +235,9 @@ class TunnelServer(StreamServer):
             except Exception as ex:
                 logger.error("Error reading from channel - %s", ex)
                 raise
-            # logger.debug("Read %s data from channel" % (size,))
+            # logger.debug("Read %s data from channel", size)
             if size == LIBSSH2_ERROR_EAGAIN:
-                self._client.poll()
+                self.client.poll()
                 continue
             elif size == 0:
                 sleep(.01)
@@ -224,7 +255,7 @@ class TunnelServer(StreamServer):
             fw_host, fw_port, self.bind_address,
             local_port)
         while channel == LIBSSH2_ERROR_EAGAIN:
-            self._client.poll()
+            self.client.poll()
             channel = self.session.direct_tcpip_ex(
                 fw_host, fw_port, self.bind_address,
                 local_port)
@@ -249,4 +280,4 @@ class TunnelServer(StreamServer):
 
 
 FORWARDER = LocalForwarder()
-FORWARDER.daemon = True
+atexit.register(FORWARDER.shutdown)

pssh/clients/ssh/single.py

@@ -16,8 +16,8 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 import logging
-
 from gevent import sleep, spawn, Timeout as GTimeout, joinall
+from gevent.socket import SHUT_RDWR
 from ssh import options
 from ssh.error_codes import SSH_AGAIN
 from ssh.exceptions import EOF
@@ -81,9 +81,11 @@ class SSHClient(BaseSSHClient):
         :type retry_delay: int or float
         :param timeout: (Optional) If provided, all commands will timeout after
           <timeout> number of seconds.
+          Also currently sets socket as well as per function timeout in some cases, see
+          function descriptions.
         :type timeout: int or float
         :param allow_agent: (Optional) set to False to disable connecting to
-          the system's SSH agent. Currently unused.
+          the system's SSH agent.
         :type allow_agent: bool
         :param identity_auth: (Optional) set to False to disable attempting to
           authenticate with default identity files from
@@ -124,10 +126,18 @@ class SSHClient(BaseSSHClient):
             ipv6_only=ipv6_only,
         )
 
-    def disconnect(self):
-        """Close socket if needed."""
-        if self.sock is not None and not self.sock.closed:
-            self.sock.close()
+    def _disconnect(self):
+        """Shutdown socket if needed.
+
+        Does not need to be called directly - called when client object is de-allocated.
+        """
+        if self.session is not None and self.sock is not None and not self.sock.closed:
+            try:
+                self.sock.shutdown(SHUT_RDWR)
+                self.sock.detach()
+            except Exception:
+                pass
+        self.sock = None
 
     def _agent_auth(self):
         self.session.userauth_agent(self.user)
@@ -201,12 +211,12 @@ class SSHClient(BaseSSHClient):
         logger.debug("Imported certificate file %s for pkey %s", self.cert_file, self.pkey)
 
     def _shell(self, channel):
-        return self._eagain(channel.request_shell)
+        return self.eagain(channel.request_shell)
 
     def _open_session(self):
         channel = self.session.channel_new()
         channel.set_blocking(0)
-        self._eagain(channel.open_session)
+        self.eagain(channel.open_session)
         return channel
 
     def open_session(self):
@@ -225,7 +235,7 @@ class SSHClient(BaseSSHClient):
             self._read_output_to_buffer, channel, stderr_buffer, is_stderr=True)
         return _stdout_reader, _stderr_reader
 
-    def execute(self, cmd, use_pty=False, channel=None):
+    def _execute(self, cmd, use_pty=False, channel=None):
         """Execute command on remote host.
 
         :param cmd: The command string to execute.
@@ -235,12 +245,13 @@ class SSHClient(BaseSSHClient):
         :type use_pty: bool
         :param channel: Channel to use. New channel is created if not provided.
         :type channel: :py:class:`ssh.channel.Channel`"""
-        channel = self.open_session() if not channel else channel
-        if use_pty:
-            self._eagain(channel.request_pty, timeout=self.timeout)
-        logger.debug("Executing command '%s'", cmd)
-        self._eagain(channel.request_exec, cmd)
-        return channel
+        with GTimeout(seconds=self.timeout, exception=Timeout):
+            channel = self.open_session() if not channel else channel
+            if use_pty:
+                self.eagain(channel.request_pty)
+            logger.debug("Executing command '%s'", cmd)
+            self.eagain(channel.request_exec, cmd)
+            return channel
 
     def _read_output_to_buffer(self, channel, _buffer, is_stderr=False):
         try:
@@ -276,7 +287,7 @@ class SSHClient(BaseSSHClient):
         if channel is None:
             return
         logger.debug("Sending EOF on channel %s", channel)
-        self._eagain(channel.send_eof, timeout=self.timeout)
+        self.eagain(channel.send_eof)
         logger.debug("Waiting for readers, timeout %s", timeout)
         with GTimeout(seconds=timeout, exception=Timeout):
             joinall((host_output.buffers.stdout.reader, host_output.buffers.stderr.reader))
@@ -311,7 +322,7 @@ class SSHClient(BaseSSHClient):
         :type channel: :py:class:`ssh.channel.Channel`
         """
         logger.debug("Closing channel")
-        self._eagain(channel.close)
+        self.eagain(channel.close)
 
     def poll(self, timeout=None):
         """ssh-python based co-operative gevent poll on session socket.
@@ -323,9 +334,9 @@ class SSHClient(BaseSSHClient):
             SSH_WRITE_PENDING,
         )
 
-    def _eagain(self, func, *args, **kwargs):
+    def eagain(self, func, *args, **kwargs):
         """Run function given and handle EAGAIN for an ssh-python session"""
         return self._eagain_errcode(func, SSH_AGAIN, *args, **kwargs)
 
-    def _eagain_write(self, write_func, data):
+    def eagain_write(self, write_func, data):
         return self._eagain_write_errcode(write_func, data, SSH_AGAIN)