amps-python-client 5.3.4.2__zip → 5.3.4.4__zip

{amps-python-client-5.3.4.2 → amps-python-client-5.3.4.4}/src/cpp/src/tcps.c

@@ -80,7 +80,62 @@ int amps_win_spin_lock(LPCRITICAL_SECTION x);
 #define AMPS_SPIN_LOCK_UNLIMITED(x) EnterCriticalSection(x)
 #define AMPS_UNLOCK(x) LeaveCriticalSection(x)
 #define AMPS_KILLLOCK(x) DeleteCriticalSection(x)
+#else
+#define GET_ERRNO (errno)
+#define SOCK_ERRORCODE(x) x
+#include <pthread.h>
+
+static pthread_mutexattr_t _mutexattr_recursive;
+#define AMPS_INITLOCK(x) { \
+    pthread_mutexattr_init(&_mutexattr_recursive); \
+    pthread_mutexattr_settype(&_mutexattr_recursive, PTHREAD_MUTEX_RECURSIVE);\
+    pthread_mutex_init(x, &_mutexattr_recursive); }
+#define AMPS_LOCK(x) pthread_mutex_lock(x)
+#define AMPS_SPIN_LOCK(x) amps_spin_lock_counted(x)
+#define AMPS_SPIN_LOCK_UNLIMITED(x) amps_spin_lock_unlimited(x)
+#define AMPS_UNLOCK(x) pthread_mutex_unlock(x)
+#define AMPS_KILLLOCK(x) pthread_mutex_destroy(x)
+#endif
+
+#ifdef AMPS_CPP_COUNT_THREADS
+#define AMPS_INC_THREAD_COUNT(x) AMPS_FETCH_ADD(x, 1)
+#else
+#define AMPS_INC_THREAD_COUNT(x)
+#endif
+
+#ifdef _WIN32
+INIT_ONCE ampsTcpsThreadKeyOnce = INIT_ONCE_STATIC_INIT;
+DWORD ampsTcpsThreadKey = TLS_OUT_OF_INDEXES;
+
+void amps_tcps_delete_key(void)
+{
+  TlsFree(ampsTcpsThreadKey);
+}
+
+BOOL CALLBACK amps_tcps_init_key(PINIT_ONCE pKeyOnce, PVOID unused, PVOID* unused2)
+{
+  if ((ampsTcpsThreadKey = TlsAlloc()) == TLS_OUT_OF_INDEXES)
+  {
+    return FALSE;
+  }
+  atexit(amps_tcps_delete_key);
+  return TRUE;
+}
+
+DWORD amps_tcps_get_thread_key(void)
+{
+  InitOnceExecuteOnce(&ampsTcpsThreadKeyOnce, amps_tcps_init_key, NULL, NULL);
+  return ampsTcpsThreadKey;
+}
+
+void amps_tcps_set_thread_key(void* pVal)
+{
+  InitOnceExecuteOnce(&ampsTcpsThreadKeyOnce, amps_tcps_init_key, NULL, NULL);
+  TlsSetValue(ampsTcpsThreadKey, pVal);
+}
+
 int tcpsThreadTimeoutMillis = 10000;
+
 #define AMPS_JOIN_DECLARE() \
   DWORD tid = (DWORD)0; \
   HANDLE tidH = (HANDLE)0
@@ -92,51 +147,90 @@ int tcpsThreadTimeoutMillis = 10000;
   { \
     if (tid != GetCurrentThreadId()) \
     { \
-      if(WAIT_TIMEOUT == WaitForSingleObject(tidH, tcpsThreadTimeoutMillis)) \
-      { \
-        TerminateThread(tidH, -1); \
-      } \
+      AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+      WaitForSingleObject(tidH, tcpsThreadTimeoutMillis); \
+      CloseHandle(tidH); \
+    } \
+    else \
+    { \
+      amps_tcps_set_thread_key((void*)tidH); \
     } \
-    CloseHandle(tidH); \
   }
 #define AMPS_SLEEP(x) \
   Sleep(x)
 #define SHUT_RDWR SD_BOTH
 #else
-#define GET_ERRNO (errno)
-#define SOCK_ERRORCODE(x) x
-#include <pthread.h>
+pthread_once_t ampsTcpsThreadKeyOnce = PTHREAD_ONCE_INIT;
+pthread_key_t ampsTcpsThreadKey;
 
-static pthread_mutexattr_t _mutexattr_recursive;
-#define AMPS_INITLOCK(x) { \
-    pthread_mutexattr_init(&_mutexattr_recursive); \
-    pthread_mutexattr_settype(&_mutexattr_recursive, PTHREAD_MUTEX_RECURSIVE);\
-    pthread_mutex_init(x, &_mutexattr_recursive); }
-#define AMPS_LOCK(x) pthread_mutex_lock(x)
-#define AMPS_SPIN_LOCK(x) amps_spin_lock_counted(x)
-#define AMPS_SPIN_LOCK_UNLIMITED(x) amps_spin_lock_unlimited(x)
+void amps_tcps_detach_thread(void* vpSelf)
+{
+  pthread_detach((pthread_t)vpSelf);
+  AMPS_INC_THREAD_COUNT(&amps_thread_detach_count);
+}
+
+void amps_tcps_delete_thread_key(void)
+{
+  pthread_key_delete(ampsTcpsThreadKey);
+}
+
+void amps_tcps_init_thread_key()
+{
+  pthread_key_create(&ampsTcpsThreadKey, amps_tcps_detach_thread);
+  atexit(amps_tcps_delete_thread_key);
+}
+
+void amps_tcps_set_thread_key(void* pVal)
+{
+  (void) pthread_once(&ampsTcpsThreadKeyOnce, amps_tcps_init_thread_key);
+  pthread_setspecific(ampsTcpsThreadKey, pVal);
+}
 
-#define AMPS_UNLOCK(x) pthread_mutex_unlock(x)
-#define AMPS_KILLLOCK(x) pthread_mutex_destroy(x)
 #define AMPS_JOIN_DECLARE() \
   pthread_t tid = (pthread_t)0;
 
 #if __STDC_VERSION__ >= 201100
 #define AMPS_JOIN(x) \
   tid = atomic_exchange((&x), (pthread_t)0); \
-  if(tid!=(pthread_t)0 && tid!=pthread_self()) \
+  if(tid != (pthread_t)0) \
+  { \
+    if(tid != pthread_self()) \
+    { \
+      AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+      pthread_join(tid, NULL); \
+    } \
+    else \
+    { \
+      amps_tcps_set_thread_key((void*)tid); \
+    } \
+  } \
+  else \
   { \
-    pthread_join(tid, NULL); \
+    /* Give recv thread a chance to exit in case it hasn't yet */ \
+    usleep(10); \
   }
 #else
 #define AMPS_JOIN(x) \
   tid = AMPS_FETCH_ADD(&x, (pthread_t)0); \
-  if(tid!=(pthread_t)0 && tid!=pthread_self()) \
+  if(tid != (pthread_t)0) \
   { \
-    if(__sync_bool_compare_and_swap(&x, tid, (pthread_t)0)) \
+    if(__sync_bool_compare_and_swap(&x, tid, 0)) \
     { \
-      pthread_join(tid, NULL); \
+      if(tid != pthread_self()) \
+      { \
+        AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+        pthread_join(tid, NULL); \
+      } \
+      else \
+      { \
+        amps_tcps_set_thread_key((void*)tid); \
+      } \
     } \
+  } \
+  else \
+  { \
+    /* Give recv thread a chance to exit in case it hasn't yet */ \
+    usleep(10); \
   }
 #endif
 
@@ -211,6 +305,47 @@ typedef struct
 }
 amps_tcps_t;
 
+// Function to close SSL* and fd that can be replaced by Python client
+int _amps_tcps_close_all(AMPS_SOCKET fd, _amps_SSL* ssl)
+{
+  int rc = 0;
+  if (ssl)
+  {
+    rc = _amps_SSL_shutdown(ssl);
+    if (rc == 0 && fd != AMPS_INVALID_SOCKET)
+    {
+      // attempt to wait for the server to send back the matching close notify.
+      struct timeval tv;
+      fd_set readfds, exceptfds;
+      tv.tv_sec = 0;
+      tv.tv_usec = 10000;
+      FD_ZERO(&readfds);
+      FD_SET(fd, &readfds);
+      FD_ZERO(&exceptfds);
+      FD_SET(fd, &exceptfds);
+#ifdef _WIN32
+      select(1, &readfds, 0, &exceptfds, &tv);
+#else
+      select(fd + 1, &readfds, 0, &exceptfds, &tv);
+#endif
+      rc = _amps_SSL_shutdown(ssl);
+    }
+  }
+  if (fd != AMPS_INVALID_SOCKET)
+  {
+    shutdown(fd, SHUT_RDWR);
+    AMPS_CLOSESOCKET(fd);
+  }
+  if (ssl)
+  {
+    _amps_SSL_free(ssl);
+  }
+  return rc;
+}
+
+// Pointer to function for closing SSL/fd that Python client redirects
+AMPSDLL int (*amps_tcps_close_all)(AMPS_SOCKET, _amps_SSL*) = _amps_tcps_close_all;
+
 const amps_char* amps_tcps_get_last_error(
   amps_handle transport)
 {
@@ -225,6 +360,13 @@ void amps_tcps_set_error(
   transport->lastErrorBuf[sizeof(transport->lastErrorBuf) - 1] = '\0';
 }
 
+// Allow python client to set transport errors during connect/close
+AMPSDLL void amps_tcps_set_external_error(void* transport,
+                                          const amps_char* buffer)
+{
+  amps_tcps_set_error((amps_tcps_t*)transport, buffer);
+}
+
 void amps_tcps_set_socket_error(
   amps_tcps_t* transport,
   int errorCode)
@@ -580,11 +722,10 @@ amps_tcps_apply_socket_properties(AMPS_SOCKET fd, const char* uri,
   return AMPS_E_OK;
 }
 
-int
-amps_tcps_apply_ssl_properties(_amps_SSL* ssl, const char* uri,
-                               size_t uriLength, amps_uri_state* uriState)
+// Retrieve server name indicator for use by SSL when connecting
+char* amps_tcps_get_sni_hostname(const char* uri, size_t uriLength,
+                                 amps_uri_state* uriState)
 {
-  int rc = 1;
   const char* key = NULL;
   size_t keyLength = 0;
   while (uriState->part_id < AMPS_URI_ERROR)
@@ -602,19 +743,92 @@ amps_tcps_apply_ssl_properties(_amps_SSL* ssl, const char* uri,
         char* hostname = (char*)malloc(uriState->part_length + 1);
         memcpy(hostname, uriState->part, uriState->part_length);
         hostname[uriState->part_length] = '\0';
-        rc = _amps_SSL_ctrl(ssl, AMPS_SSL_CTRL_SET_TLSEXT_HOSTNAME, AMPS_TLSEXT_NAMETYPE_host_name, hostname);
-        free(hostname);
-        if (rc != 1)
-        {
-          return rc;
-        }
+        return hostname;
       }
     }
   }
-  return rc;
+  return NULL;
+}
+
+// Normal initialization and connection of SSL around socket
+_amps_SSL* _amps_tcps_get_socket_and_ssl(AMPS_SOCKET fd_,
+                                         const char* address_, int port_,
+                                         struct addrinfo* pAddrInfo_,
+                                         char* hostname_, void* me_)
+{
+  amps_tcps_t* transport = (amps_tcps_t*)me_;
+  /* On first use, initialize an SSL context if one hasn't been created. */
+  if (!_amps_ssl_ctx)
+  {
+    if (amps_ssl_init(NULL))
+    {
+      return NULL;
+    }
+  }
+  // Create SSL wrapper for the socket
+  _amps_ERR_clear_error();
+  _amps_SSL* ssl = _amps_SSL_new(_amps_ssl_ctx);
+
+  int rc = _amps_SSL_set_fd(ssl, (int)(fd_));
+  int sysError = GET_ERRNO;
+  if (rc != 1)
+  {
+    transport->ssl = ssl;
+    amps_tcps_set_ssl_error(transport, rc, sysError);
+    _amps_SSL_free(ssl);
+    ssl = (_amps_SSL*)0;
+    transport->ssl = (_amps_SSL*)0;
+    return NULL;
+  }
+
+  // Connect socket
+  rc = connect(fd_, pAddrInfo_->ai_addr,
+               (socklen_t)pAddrInfo_->ai_addrlen);
+  sysError = GET_ERRNO;
+  if (rc == -1)
+  {
+    amps_tcps_set_socket_error(transport, sysError);
+    _amps_SSL_free(ssl);
+    return NULL;
+  }
+
+  // Set server name indicator if used
+  if (hostname_)
+  {
+    rc = _amps_SSL_ctrl(ssl, AMPS_SSL_CTRL_SET_TLSEXT_HOSTNAME,
+                        AMPS_TLSEXT_NAMETYPE_host_name, hostname_);
+    sysError = GET_ERRNO;
+    if (rc != 1)
+    {
+      amps_tcps_set_ssl_error(transport, rc, sysError);
+      _amps_SSL_free(ssl);
+      return NULL;
+    }
+  }
+
+  _amps_SSL_ctrl(ssl, AMPS_SSL_CTRL_MODE, AMPS_SSL_AUTO_RETRY, NULL);
+
+  // Complete SSL connect/handshake
+  rc = _amps_SSL_connect(ssl);
+  sysError = GET_ERRNO;
+  if (rc != 1)
+  {
+    amps_tcps_set_ssl_error(transport, rc, sysError);
+    _amps_SSL_free(ssl);
+    return NULL;
+  }
+  return ssl;
 }
 
+// Pointer to init/connect function for Python client to override
+AMPSDLL _amps_SSL* (*amps_tcps_get_socket_and_ssl)(AMPS_SOCKET fd_,
+                                                   const char* address_, int port_,
+                                                   struct addrinfo* pAddrInfo_,
+                                                   char* hostname_, void* me_) =
+                                                   _amps_tcps_get_socket_and_ssl;
+
 amps_result amps_tcps_set_idle_time(amps_handle transport, int idleTimeMillis);
+
 amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
 {
   AMPS_JOIN_DECLARE();
@@ -654,27 +868,15 @@ amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
   ssl = (_amps_SSL*)AMPS_IEX_GET(&me->ssl, 0);
 
   /* were we previously connected?  go ahead and close that. */
-  if (ssl)
-  {
-    rc = _amps_SSL_shutdown(ssl);
-    sysError = GET_ERRNO;
-    if (rc < 0)
-    {
-      amps_tcps_set_ssl_error(me, rc, sysError);
-    }
-  }
-  if (fd != AMPS_INVALID_SOCKET)
-  {
-    /* this is a reconnect */
-    shutdown(fd, SHUT_RDWR);
-    closesocket(fd);
-    fd = AMPS_INVALID_SOCKET;
-  }
+  // Use our close function to shutdown socket and SSL
+  rc = amps_tcps_close_all(fd, ssl);
+  sysError = GET_ERRNO;
   AMPS_JOIN(me->thread);
-  if (ssl)
+  fd = AMPS_INVALID_SOCKET;
+  ssl = NULL;
+  if (rc < 0)
   {
-    _amps_SSL_free(ssl);
-    ssl = 0;
+    amps_tcps_set_ssl_error(me, rc, sysError);
   }
 
   /* parse out the address */
@@ -779,106 +981,64 @@ amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
     goto error;
   }
 
-  me->fd = socket(pAddrInfo->ai_family, pAddrInfo->ai_socktype, pAddrInfo->ai_protocol);
+  // Get the socket we'll use
+  fd = socket(pAddrInfo->ai_family, pAddrInfo->ai_socktype, pAddrInfo->ai_protocol);
   sysError = GET_ERRNO;
-  if (me->fd == AMPS_INVALID_SOCKET)
+  if (fd == AMPS_INVALID_SOCKET)
   {
     freeaddrinfo(pAddrInfo);
     amps_tcps_set_socket_error(me, sysError);
     result = AMPS_E_CONNECTION_REFUSED;
     goto error;
   }
+  AMPS_IEX(&me->fd, fd);
 
   /* apply socket properties */
-  if (amps_tcps_apply_socket_properties(me->fd, address, address_length, &uri_state_socket_props) != AMPS_E_OK)
+  if (me->destroying
+      || amps_tcps_apply_socket_properties(me->fd, address, address_length, &uri_state_socket_props) != AMPS_E_OK)
   {
     freeaddrinfo(pAddrInfo);
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
+    if (!me->destroying)
+    {
+      fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
+    }
     amps_tcps_set_error(me, "The URI specified invalid properties.");
     result = AMPS_E_URI;
     goto error;
   }
 #ifdef __APPLE__
   rc = 1; /* Set NOSIGPIPE to ON */
-  if (setsockopt(me->fd, SOL_SOCKET, SO_NOSIGPIPE, &rc, sizeof(rc)) < 0)
+  if (me->destroying
+      || setsockopt(me->fd, SOL_SOCKET, SO_NOSIGPIPE, &rc, sizeof(rc)) < 0)
   {
     freeaddrinfo(pAddrInfo);
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
+    if (!me->destroying)
+    {
+      fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
+    }
     amps_tcps_set_error(me, "Failed to set no SIGPIPE.");
     result = AMPS_E_CONNECTION;
     goto error;
   }
 #endif
 
-  rc = connect(me->fd, pAddrInfo->ai_addr,
-               (socklen_t)pAddrInfo->ai_addrlen);
-  sysError = GET_ERRNO;
+  // Call function to do SSL init and connect
+  char* hostname = amps_tcps_get_sni_hostname(address, address_length, &uri_state_ssl_props);
+  int portNum = atoi(port);
+  me->ssl = amps_tcps_get_socket_and_ssl((AMPS_SOCKET)(me->fd), host, portNum, pAddrInfo, hostname, (void*)me);
   freeaddrinfo(pAddrInfo);
-  if (rc == -1)
-  {
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
-    amps_tcps_set_socket_error(me, sysError);
-    result = AMPS_E_CONNECTION_REFUSED;
-    goto error;
-  }
-
-  /* On first use, initialize an SSL context if one hasn't been created. */
-  if (!_amps_ssl_ctx)
+  if (me->destroying || me->ssl == 0)
   {
-    rc = amps_ssl_init(NULL);
-    if (rc)
+    if (!me->destroying)
     {
       fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
-      amps_tcps_set_error(me, "Could not initialize an SSL context.");
-      result = AMPS_E_SSL;
-      goto error;
     }
-  }
-  /* Handshake via SSL */
-  _amps_ERR_clear_error();
-  if (!me->ssl)
-  {
-    me->ssl = _amps_SSL_new(_amps_ssl_ctx);
-  }
-  if (me->ssl == 0)
-  {
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
     amps_tcps_set_error_stack_error(me);
-    result = AMPS_E_SSL;
-    goto error;
-  }
-
-  rc = _amps_SSL_set_fd(me->ssl, (int)me->fd);
-  sysError = GET_ERRNO;
-  if (rc != 1)
-  {
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
-    ssl = (_amps_SSL*)AMPS_IEX_GET(&me->ssl, 0);
-    amps_tcps_set_ssl_error(me, rc, sysError);
-    result = AMPS_E_SSL;
-    goto error;
-  }
-
-  rc = amps_tcps_apply_ssl_properties(me->ssl, address, address_length, &uri_state_ssl_props);
-  sysError = GET_ERRNO;
-  if (rc != 1)
-  {
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
-    ssl = (_amps_SSL*)AMPS_IEX_GET(&me->ssl, 0);
-    amps_tcps_set_ssl_error(me, rc, sysError);
-    result = AMPS_E_SSL;
-    goto error;
-  }
-
-  _amps_SSL_ctrl(me->ssl, AMPS_SSL_CTRL_MODE, AMPS_SSL_AUTO_RETRY, NULL);
-
-  rc = _amps_SSL_connect(me->ssl);
-  sysError = GET_ERRNO;
-  if (rc != 1)
-  {
-    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
-    ssl = (_amps_SSL*)AMPS_IEX_GET(&me->ssl, 0);
-    amps_tcps_set_ssl_error(me, rc, sysError);
+    // The error string should already be set, but just in case
+    if (!amps_tcps_get_last_error(me))
+    {
+      amps_tcps_set_error(me, "Could not initialize an SSL context or SSL.");
+    }
     result = AMPS_E_SSL;
     goto error;
   }
@@ -904,6 +1064,7 @@ amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
    * important to do it in two steps, since we use me->thread as a poor-man's cancellation mechanism */
   if (me->threadH)
   {
+    AMPS_INC_THREAD_COUNT(&amps_thread_create_count);
     ResumeThread(me->threadH);
   }
   else
@@ -926,6 +1087,7 @@ amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
     result = AMPS_E_MEMORY;
     goto error;
   }
+  AMPS_INC_THREAD_COUNT(&amps_thread_create_count);
 #endif
   for (i = 0; i < AMPS_THREAD_START_TIMEOUT && me->threadCreatedCallbackResult == -1; ++i)
   {
@@ -941,30 +1103,18 @@ amps_result amps_tcps_connect(amps_handle transport, const amps_char* address)
   {
     fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
     ssl = (_amps_SSL*)AMPS_IEX_GET(&me->ssl, 0);
+    AMPS_JOIN(me->thread);
     amps_tcps_set_error(me, "Thread creation callback failed");
   }
 
 error:
   if (result != AMPS_E_OK)
   {
-    if (ssl)
-    {
-      rc = _amps_SSL_shutdown(ssl);
-      sysError = GET_ERRNO;
-      if (rc < 0)
-      {
-        amps_tcps_set_ssl_error(me, rc, sysError);
-      }
-    }
-    if (fd != AMPS_INVALID_SOCKET)
-    {
-      shutdown(fd, SHUT_RDWR);
-      closesocket(fd);
-    }
-    if (ssl)
+    // Use our close function to shutdown socket and SSL
+    int rc = amps_tcps_close_all(fd, ssl);
+    if (rc < 0)
     {
-      _amps_SSL_free(ssl);
-      ssl = 0;
+      amps_tcps_set_ssl_error(me, rc, sysError);
     }
 #ifdef _WIN32
     me->thread = 0;
@@ -1032,41 +1182,12 @@ void amps_tcps_close(amps_handle transport)
 #ifndef _WIN32
   pthread_cleanup_push(amps_cleanup_unlock_mutex, (void*)&me->lock);
 #endif
-  if (ssl)
+  // Use our close function to shutdown socket and SSL
+  int rc = amps_tcps_close_all(fd, ssl);
+  if (rc < 0)
   {
-    int rc = _amps_SSL_shutdown(ssl);
     int sysError = GET_ERRNO;
-    if (rc == 0 && fd != AMPS_INVALID_SOCKET)
-    {
-      // attempt to wait for the server to send back the matching close notify.
-      struct timeval tv;
-      fd_set readfds, exceptfds;
-      tv.tv_sec = 0;
-      tv.tv_usec = 10000;
-      FD_ZERO(&readfds);
-      FD_SET(fd, &readfds);
-      FD_ZERO(&exceptfds);
-      FD_SET(fd, &exceptfds);
-#ifdef _WIN32
-      select(1, &readfds, 0, &exceptfds, &tv);
-#else
-      select(fd + 1, &readfds, 0, &exceptfds, &tv);
-#endif
-      rc = _amps_SSL_shutdown(ssl);
-    }
-    if (rc < 0)
-    {
-      amps_tcps_set_ssl_error(me, rc, sysError);
-    }
-  }
-  if (fd != AMPS_INVALID_SOCKET)
-  {
-    shutdown(fd, SHUT_RDWR);
-    closesocket(fd);
-  }
-  if (ssl)
-  {
-    _amps_SSL_free(ssl);
+    amps_tcps_set_ssl_error(me, rc, sysError);
   }
   AMPS_UNLOCK(&me->lock);
 #ifndef _WIN32
@@ -1103,41 +1224,12 @@ void amps_tcps_destroy(amps_handle transport)
   AMPS_IEX_LONG(&me->destroying, 1);
   AMPS_IEX_LONG(&me->disconnecting, 1);
 #endif
-  if (ssl)
+  // Use our close function to shutdown socket and SSL
+  int rc = amps_tcps_close_all(fd, ssl);
+  if (rc < 0)
   {
-    int rc = _amps_SSL_shutdown(ssl);
     int sysError = GET_ERRNO;
-    if (rc == 0 && fd != AMPS_INVALID_SOCKET)
-    {
-      // attempt to wait for the server to send back the matching close notify.
-      struct timeval tv;
-      fd_set readfds, exceptfds;
-      tv.tv_sec = 0;
-      tv.tv_usec = 10000;
-      FD_ZERO(&readfds);
-      FD_SET(fd, &readfds);
-      FD_ZERO(&exceptfds);
-      FD_SET(fd, &exceptfds);
-#ifdef _WIN32
-      select(1, &readfds, 0, &exceptfds, &tv);
-#else
-      select(fd + 1, &readfds, 0, &exceptfds, &tv);
-#endif
-      rc = _amps_SSL_shutdown(ssl);
-    }
-    if (rc < 0)
-    {
-      amps_tcps_set_ssl_error(me, rc, sysError);
-    }
-  }
-  if (fd != AMPS_INVALID_SOCKET)
-  {
-    shutdown(fd, SHUT_RDWR);
-    closesocket(fd);
-  }
-  if (ssl)
-  {
-    _amps_SSL_free(ssl);
+    amps_tcps_set_ssl_error(me, rc, sysError);
   }
 
   AMPS_UNLOCK(&me->lock);
@@ -1176,42 +1268,12 @@ amps_result amps_tcps_handle_disconnect(
 #ifndef _WIN32
   pthread_cleanup_pop(0);
 #endif
-  /* close what we have right now. */
-  if (ssl)
+  // Use our close function to shutdown socket and SSL
+  int rc = amps_tcps_close_all(fd, ssl);
+  if (rc < 0)
   {
-    int rc = _amps_SSL_shutdown(ssl);
     int sysError = GET_ERRNO;
-    if (rc == 0 && fd != AMPS_INVALID_SOCKET)
-    {
-      // attempt to wait for the server to send back the matching close notify.
-      struct timeval tv;
-      fd_set readfds, exceptfds;
-      tv.tv_sec = 0;
-      tv.tv_usec = 10000;
-      FD_ZERO(&readfds);
-      FD_SET(fd, &readfds);
-      FD_ZERO(&exceptfds);
-      FD_SET(fd, &exceptfds);
-#ifdef _WIN32
-      select(1, &readfds, 0, &exceptfds, &tv);
-#else
-      select(fd + 1, &readfds, 0, &exceptfds, &tv);
-#endif
-      rc = _amps_SSL_shutdown(ssl);
-    }
-    if (rc < 0)
-    {
-      amps_tcps_set_ssl_error(me, rc, sysError);
-    }
-  }
-  if (fd != AMPS_INVALID_SOCKET)
-  {
-    shutdown(fd, SHUT_RDWR);
-    closesocket(fd);
-  }
-  if (ssl)
-  {
-    _amps_SSL_free(ssl);
+    amps_tcps_set_ssl_error(me, rc, sysError);
   }
 
   /* first let waiters know that we've failed. */
@@ -1249,23 +1311,27 @@ amps_result amps_tcps_handle_disconnect(
     goto error;
   }
 
-#ifndef _WIN32
+#ifdef _WIN32
+  HANDLE tidH = (AMPS_CURRENT_THREAD() == me->thread) ? me->threadH : NULL;
+#else
   pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancelState);
+  void* vpTid = (void*)(pthread_self());
 #endif
   /* Call the disconect handler. */
   result = me->disconnectHandler(me, me->disconnectHandlerUserData);
-#ifndef _WIN32
-  pthread_setcancelstate(cancelState, &unusedCancelState);
-#endif
   if (result == AMPS_E_OK)
   {
-    result = AMPS_E_RETRY;
+#ifdef _WIN32
+    if (tidH)
+    {
+      amps_tcps_set_thread_key(tidH);
+    }
+#else
+    amps_tcps_set_thread_key(vpTid);
+#endif
   }
 
 error:
-#ifndef _WIN32
-  pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancelState);
-#endif
   AMPS_UNLOCK(&me->lock);
 #ifndef _WIN32
   pthread_cleanup_pop(0);
@@ -1460,12 +1526,13 @@ int amps_tcps_locked_read(amps_tcps_t* me_,
     FD_SET(fd, &exceptfds);
     AMPS_UNLOCK(&me_->sendLock);
 #ifdef _WIN32
-    if (select(1, &readfds, 0, &exceptfds, &tv) == 0)
+    rc  = select(1, &readfds, 0, &exceptfds, &tv);
 #else
-    if (select(fd + 1, &readfds, 0, &exceptfds, &tv) == 0)
+    rc = select(fd + 1, &readfds, 0, &exceptfds, &tv);
 #endif
+    if (rc <= 0)
     {
-      return 0;
+      return rc;
     }
     AMPS_LOCK(&me_->sendLock);
     if (me_->disconnecting || fd != me_->fd || !me_->ssl
@@ -1492,6 +1559,7 @@ int amps_tcps_locked_read(amps_tcps_t* me_,
   {
     if (*pErrorcode_ == AMPS_SSL_ERROR_WANT_READ)
     {
+      *pErrorcode_ = 0;
       return 0;
     }
     // End of stream
@@ -1503,12 +1571,12 @@ int amps_tcps_locked_read(amps_tcps_t* me_,
 #ifdef _WIN32
 DWORD amps_tcps_threaded_reader(void* userData)
 {
-  DWORD self = GetCurrentThreadId();
-  HANDLE selfH = GetCurrentThread();
+  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
+  HANDLE tidH = GetCurrentThread();
   amps_tcps_t* me = (amps_tcps_t*)userData;
   if (me->threadCreatedCallback)
   {
-    amps_result result = me->threadCreatedCallback(selfH,
+    amps_result result = me->threadCreatedCallback(tidH,
                                                    me->threadCreatedCallbackUserData);
     AMPS_IEX(&me->threadCreatedCallbackResult, (long)result);
     if (result != AMPS_E_OK)
@@ -1525,8 +1593,9 @@ void* amps_tcps_threaded_reader(void* userData)
 {
   int unusedCancelState = 0;
   amps_tcps_t* me = (amps_tcps_t*)userData;
+  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
   /* Wait for pthread_create to set me->thread */
-  while (me->thread != pthread_self()
+  while (me->thread != tid
          && !me->disconnecting
          && !me->destroying)
   {
@@ -1547,17 +1616,18 @@ void* amps_tcps_threaded_reader(void* userData)
     AMPS_IEX(&me->threadCreatedCallbackResult, (long)AMPS_E_OK);
   }
 #endif
-  unsigned char* buffer, *newBuffer, *end, *readPoint, *parsePoint;
+  unsigned char* buffer = 0, *newBuffer = 0, *end = 0, *readPoint = 0;
+  unsigned char* parsePoint = 0;
 
-  unsigned int msglenN;
-  unsigned long msglen, currentPosition, bytesRead = 0;
-  int received;
+  unsigned int msglenN = 0;
+  unsigned long msglen = 0, currentPosition = 0, bytesRead = 0;
+  int received = 0;
   int cancelState = 0;
   const size_t BUFFER_SIZE = 16 * 1024;
   const unsigned MAX_MESSAGE_SIZE = 1024 * 1024 * 1024;
   amps_uint64_t lastReadTime = 0;
   amps_uint64_t lastIdleTime = 0;
-  amps_handle   message;
+  amps_handle   message = 0;
 
   /* capture the connection version we are using now. */
   unsigned connectionVersion = me->connectionVersion;
@@ -1626,7 +1696,7 @@ void* amps_tcps_threaded_reader(void* userData)
     else
     {
       // errorcode could be an ssl error code, so check for errno
-      int sysError = GET_ERRNO;
+      int sysError = received ? GET_ERRNO : 0;
       if (sysError)
       {
         errorcode = sysError;
@@ -1641,7 +1711,8 @@ void* amps_tcps_threaded_reader(void* userData)
            && ((errorcode != SOCK_ERRORCODE(ETIMEDOUT)
                 && errorcode != SOCK_ERRORCODE(EWOULDBLOCK))
               || me->idleTimeMillis == 0))
-           || (me->readTimeoutMillis && amps_now() - lastReadTime > me->readTimeoutMillis))
+           || (me->readTimeoutMillis
+              && amps_now() - lastReadTime > me->readTimeoutMillis))
       {
         if (!me->disconnecting)
         {
@@ -1757,33 +1828,44 @@ cleanup:
 #endif
   amps_message_destroy(message);
   free(buffer);
-  if (me->destroying)
-  {
-    return 0;
-  }
   // If exiting as the reader thread, clear me->thread
   // Unless me-thread is 0, we're not getting joined, so detach
-  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
   if (me->threadExitCallback)
   {
     me->threadExitCallback(tid, me->threadExitCallbackUserData);
   }
 #if __STDC_VERSION__ >= 201100
-  if (atomic_compare_exchange_strong(&(me->thread), &tid, 0))
+  AMPS_THREAD_ID mytid = tid;
+  bool swapped = atomic_compare_exchange_strong(&(me->thread), &tid, 0);
+  if (swapped || me->thread != 0)
 #elif defined(_WIN32)
+  // This returns the initial value of me->thread
   if (InterlockedCompareExchange((volatile unsigned int*)(&(me->thread)),
-                                 (unsigned int)tid, (unsigned int)0))
+                                 (unsigned int)tid, (unsigned int)0) != 0)
 #else
+  AMPS_THREAD_ID mytid = tid;
   int swapped = (int)__sync_bool_compare_and_swap(&(me->thread), tid, 0);
   if (swapped || me->thread != 0)
 #endif
   {
+    AMPS_INC_THREAD_COUNT(&amps_thread_detach_count);
 #ifdef _WIN32
     CloseHandle((HANDLE)(AMPS_IEX(&(me->threadH), 0)));
 #else
-    pthread_detach(tid);
+    pthread_detach(mytid);
 #endif
+    amps_tcps_set_thread_key(NULL);
   }
+#ifdef _WIN32
+  else
+  {
+    tidH = TlsGetValue(amps_tcps_get_thread_key());
+    if (tidH != NULL)
+    {
+      CloseHandle(tidH);
+    }
+  }
+#endif
 
   return 0;
 }
@@ -1798,6 +1880,12 @@ amps_tcps_attempt_reconnect(amps_handle transport, unsigned version)
     version = me->connectionVersion;
   }
   res = amps_tcps_handle_disconnect(me, version);
+  if (res == AMPS_E_OK)
+  {
+    // We don't want non-receive threads detaching
+    amps_tcps_set_thread_key(NULL);
+    res = AMPS_E_RETRY;
+  }
   return res;
 }