amps-python-client 5.3.4.2__zip → 5.3.4.3__zip

{amps-python-client-5.3.4.2 → amps-python-client-5.3.4.3}/src/cpp/src/tcp.c

@@ -91,25 +91,6 @@ BOOL amps_win_spin_lock(LPCRITICAL_SECTION x)
 #define AMPS_SPIN_LOCK_UNLIMITED(x) EnterCriticalSection(x)
 #define AMPS_UNLOCK(x) LeaveCriticalSection(x)
 #define AMPS_KILLLOCK(x) DeleteCriticalSection(x)
-int tcpThreadTimeoutMillis = 10000;
-#define AMPS_JOIN_DECLARE() \
-  DWORD tid = (DWORD)0; \
-  HANDLE tidH = (HANDLE)0
-
-#define AMPS_JOIN(x) \
-  tid = (DWORD)AMPS_IEX(&x, 0); \
-  tidH = (HANDLE)AMPS_IEX(&x##H, 0); \
-  if (tid != 0) \
-  { \
-    if (tid != GetCurrentThreadId()) \
-    { \
-      WaitForSingleObject(tidH, tcpThreadTimeoutMillis); \
-    } \
-    CloseHandle(tidH); \
-  }
-#define AMPS_SLEEP(x) \
-  Sleep(x)
-#define SHUT_RDWR SD_BOTH
 #else
 #define GET_ERRNO (errno)
 #define SOCK_ERRORCODE(x) x
@@ -166,25 +147,182 @@ void amps_cleanup_free_buffer(void* data)
 
 #define AMPS_UNLOCK(x) pthread_mutex_unlock(x)
 #define AMPS_KILLLOCK(x) pthread_mutex_destroy(x)
+#endif
+
+#ifdef AMPS_CPP_COUNT_THREADS
+#if __STDC_VERSION__ >= 201100
+  _Atomic size_t amps_thread_create_count = 0;
+  _Atomic size_t amps_thread_join_count = 0;
+  _Atomic size_t amps_thread_detach_count = 0;
+#else
+  volatile size_t amps_thread_create_count = 0;
+  volatile size_t amps_thread_join_count = 0;
+  volatile size_t amps_thread_detach_count = 0;
+#endif
+
+#define AMPS_COUNT_THREADS_LOG(x, y) \
+  fprintf(stderr, x, y); fflush(stderr);
+
+#define AMPS_COUNT_THREADS_LOG2(x, y, z) \
+  fprintf(stderr, x, y, z); fflush(stderr);
+
+size_t amps_get_thread_create_count(void)
+{
+  return AMPS_FETCH_ADD(&amps_thread_create_count, 0);
+}
+
+size_t amps_get_thread_join_count(void)
+{
+  return AMPS_FETCH_ADD(&amps_thread_join_count, 0);
+}
+
+size_t amps_get_thread_detach_count(void)
+{
+  return AMPS_FETCH_ADD(&amps_thread_detach_count, 0);
+}
+
+#define AMPS_INC_THREAD_COUNT(x) AMPS_FETCH_ADD(x, 1)
+#else
+#define AMPS_INC_THREAD_COUNT(x)
+#define AMPS_COUNT_THREADS_LOG(x, y)
+#define AMPS_COUNT_THREADS_LOG2(x, y, z)
+#endif
+
+#ifdef _WIN32
+INIT_ONCE ampsTcpThreadKeyOnce = INIT_ONCE_STATIC_INIT;
+DWORD ampsTcpThreadKey = TLS_OUT_OF_INDEXES;
+
+void amps_tcp_delete_key(void)
+{
+  TlsFree(ampsTcpThreadKey);
+}
+
+BOOL CALLBACK amps_tcp_init_key(PINIT_ONCE pKeyOnce, PVOID unused, PVOID* unused2)
+{
+  if ((ampsTcpThreadKey = TlsAlloc()) == TLS_OUT_OF_INDEXES)
+  {
+    return FALSE;
+  }
+  atexit(amps_tcp_delete_key);
+  return TRUE;
+}
+
+DWORD amps_tcp_get_thread_key(void)
+{
+  InitOnceExecuteOnce(&ampsTcpThreadKeyOnce, amps_tcp_init_key, NULL, NULL);
+  return ampsTcpThreadKey;
+}
+
+void amps_tcp_set_thread_key(void* pVal)
+{
+  InitOnceExecuteOnce(&ampsTcpThreadKeyOnce, amps_tcp_init_key, NULL, NULL);
+  TlsSetValue(ampsTcpThreadKey, pVal);
+}
+
+int tcpThreadTimeoutMillis = 10000;
+
+#define AMPS_JOIN_DECLARE() \
+  DWORD tid = (DWORD)0; \
+  HANDLE tidH = (HANDLE)0
+
+#define AMPS_JOIN(x) \
+  tid = (DWORD)AMPS_IEX(&x, 0); \
+  tidH = (HANDLE)AMPS_IEX(&x##H, 0); \
+  if (tid != 0) \
+  { \
+    if (tid != GetCurrentThreadId()) \
+    { \
+      AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+      WaitForSingleObject(tidH, tcpThreadTimeoutMillis); \
+      CloseHandle(tidH); \
+    } \
+    else \
+    { \
+      amps_tcp_set_thread_key((void*)tidH); \
+    } \
+  } \
+  else \
+  { \
+    /* Give recv thread a chance to exit in case it hasn't yet */ \
+    Sleep(1); \
+  }
+#define AMPS_SLEEP(x) \
+  Sleep(x)
+#define SHUT_RDWR SD_BOTH
+#else
+pthread_once_t ampsTcpThreadKeyOnce = PTHREAD_ONCE_INIT;
+pthread_key_t ampsTcpThreadKey;
+
+void amps_tcp_detach_thread(void* vpSelf)
+{
+  AMPS_COUNT_THREADS_LOG("Key destruction detach of %lu\n", (unsigned long)vpSelf)
+  pthread_detach((pthread_t)vpSelf);
+  AMPS_INC_THREAD_COUNT(&amps_thread_detach_count);
+}
+
+void amps_tcp_delete_thread_key(void)
+{
+  pthread_key_delete(ampsTcpThreadKey);
+}
+
+void amps_tcp_init_thread_key()
+{
+  pthread_key_create(&ampsTcpThreadKey, amps_tcp_detach_thread);
+  atexit(amps_tcp_delete_thread_key);
+}
+
+void amps_tcp_set_thread_key(void* pVal)
+{
+  (void) pthread_once(&ampsTcpThreadKeyOnce, amps_tcp_init_thread_key);
+  pthread_setspecific(ampsTcpThreadKey, pVal);
+}
+
 #define AMPS_JOIN_DECLARE() \
   pthread_t tid = 0;
 
 #if __STDC_VERSION__ >= 201100
 #define AMPS_JOIN(x) \
   tid = atomic_exchange((&x), (pthread_t)0); \
-  if(tid != (pthread_t)0 && tid != pthread_self()) \
+  if(tid != (pthread_t)0) \
   { \
-    pthread_join(tid, NULL); \
+    if(tid != pthread_self()) \
+    { \
+      AMPS_COUNT_THREADS_LOG2("Join of %lu from %lu\n", tid, pthread_self()) \
+      AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+      pthread_join(tid, NULL); \
+    } \
+    else \
+    { \
+      amps_tcp_set_thread_key((void*)tid); \
+    } \
+  } \
+  else \
+  { \
+    /* Give recv thread a chance to exit in case it hasn't yet */ \
+    usleep(10); \
   }
 #else
 #define AMPS_JOIN(x) \
   tid = AMPS_FETCH_ADD(&x, 0); \
-  if(tid != 0) \
+  if(tid != (pthread_t)0) \
   { \
-    if(tid != pthread_self() && __sync_bool_compare_and_swap(&x, tid, 0)) \
+    if(__sync_bool_compare_and_swap(&x, tid, 0)) \
     { \
-      pthread_join(tid, NULL); \
+      if(tid != pthread_self()) \
+      { \
+        AMPS_INC_THREAD_COUNT(&amps_thread_join_count); \
+        pthread_join(tid, NULL); \
+      } \
+      else \
+      { \
+        amps_tcp_set_thread_key((void*)tid); \
+      } \
     } \
+  } \
+  else \
+  { \
+    /* Give recv thread a chance to exit in case it hasn't yet */ \
+    usleep(10); \
   }
 #endif
 
@@ -786,6 +924,7 @@ amps_result amps_tcp_connect(amps_handle transport, const amps_char* address)
    * important to do it in two steps, since we use me->thread as a poor-man's cancellation mechanism */
   if (me->threadH)
   {
+    AMPS_INC_THREAD_COUNT(&amps_thread_create_count);
     ResumeThread(me->threadH);
   }
   else
@@ -806,6 +945,8 @@ amps_result amps_tcp_connect(amps_handle transport, const amps_char* address)
     result = AMPS_E_MEMORY;
     goto error;
   }
+  AMPS_COUNT_THREADS_LOG2("Create of %lu from %lu\n", me->thread, pthread_self())
+  AMPS_INC_THREAD_COUNT(&amps_thread_create_count);
 #endif
   for (i = 0 ; i < AMPS_THREAD_START_TIMEOUT && me->threadCreatedCallbackResult == -1L; ++i)
   {
@@ -815,6 +956,8 @@ amps_result amps_tcp_connect(amps_handle transport, const amps_char* address)
   if (me->threadCreatedCallbackResult == -1L)
   {
     amps_tcp_set_error(me, "Thread created callback failed to return in a timely manner or returned -1.");
+    fd = AMPS_IEX_GET(&me->fd, AMPS_INVALID_SOCKET);
+    AMPS_JOIN(me->thread);
     result = AMPS_E_MEMORY;
   }
 error:
@@ -988,22 +1131,26 @@ amps_result amps_tcp_handle_disconnect(
     goto error;
   }
 
-#ifndef _WIN32
+#ifdef _WIN32
+  HANDLE tidH = (AMPS_CURRENT_THREAD() == me->thread) ? me->threadH : NULL;
+#else
   pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancelState);
+  void* vpTid = (void*)(pthread_self());
 #endif
-  /* Call the disconect handler. */
+  /* Call the disconnect handler. */
   result = me->disconnectHandler(me, me->disconnectHandlerUserData);
-#ifndef _WIN32
-  pthread_setcancelstate(cancelState, &unusedCancelState);
-#endif
   if (result == AMPS_E_OK)
   {
-    result = AMPS_E_RETRY;
+#ifdef _WIN32
+    if (tidH)
+    {
+      amps_tcp_set_thread_key(tidH);
+    }
+#else
+    amps_tcp_set_thread_key(vpTid);
+#endif
   }
 error:
-#ifndef _WIN32
-  pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancelState);
-#endif
   AMPS_UNLOCK(&me->lock);
 #ifndef _WIN32
   pthread_cleanup_pop(0);
@@ -1123,7 +1270,7 @@ amps_tcp_send(amps_handle transport,
 DWORD amps_tcp_threaded_reader(void* userData)
 {
   amps_tcp_t* me = (amps_tcp_t*)userData;
-  DWORD self = GetCurrentThreadId();
+  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
   if (me->threadCreatedCallback)
   {
     amps_result result = me->threadCreatedCallback(me->threadH,
@@ -1142,8 +1289,9 @@ DWORD amps_tcp_threaded_reader(void* userData)
 void* amps_tcp_threaded_reader(void* userData)
 {
   amps_tcp_t* me = (amps_tcp_t*)userData;
+  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
   /* Wait for pthread_create to set me->thread */
-  while (me->thread != pthread_self()
+  while (me->thread != tid
          && !me->disconnecting
          && !me->destroying)
   {
@@ -1212,7 +1360,7 @@ void* amps_tcp_threaded_reader(void* userData)
   while (connectionVersion == me->connectionVersion
          && !me->disconnecting
          && fd == me->fd
-         && me->thread == AMPS_CURRENT_THREAD()  /* our cancellation mechanism on win32 */
+         && me->thread == tid  /* our cancellation mechanism on win32 */
         )
   {
     if (me->destroying)
@@ -1386,33 +1534,45 @@ cleanup:
 #endif
   amps_message_destroy(message);
   free(buffer);
-  if (me->destroying)
-  {
-    return 0;
-  }
   // If exiting as the reader thread, clear me->thread
   // Unless me-thread is 0, we're not getting joined, so detach
-  AMPS_THREAD_ID tid = AMPS_CURRENT_THREAD();
   if (me->threadExitCallback)
   {
     me->threadExitCallback(tid, me->threadExitCallbackUserData);
   }
 #if __STDC_VERSION__ >= 201100
-  if (atomic_compare_exchange_strong(&(me->thread), &tid, 0))
+  AMPS_THREAD_ID mytid = tid;
+  bool swapped = atomic_compare_exchange_strong(&(me->thread), &tid, 0);
+  if (swapped || me->thread != 0)
 #elif defined(_WIN32)
+  // This returns the initial value of me->thread
   if (InterlockedCompareExchange((volatile unsigned int*)(&(me->thread)),
-                                 (unsigned int)tid, (unsigned int)0))
+                                 (unsigned int)tid, (unsigned int)0) != 0)
 #else
+  AMPS_THREAD_ID mytid = tid;
   int swapped = (int)__sync_bool_compare_and_swap(&(me->thread), tid, 0);
   if (swapped || me->thread != 0)
 #endif
   {
+    AMPS_COUNT_THREADS_LOG("Detach of %lu\n", mytid)
+    AMPS_INC_THREAD_COUNT(&amps_thread_detach_count);
 #ifdef _WIN32
     CloseHandle((HANDLE)(AMPS_IEX(&(me->threadH), 0)));
 #else
-    pthread_detach(tid);
+    pthread_detach(mytid);
 #endif
+    amps_tcp_set_thread_key(NULL);
+  }
+#ifdef _WIN32
+  else
+  {
+    HANDLE tidH = TlsGetValue(amps_tcp_get_thread_key());
+    if (tidH != NULL)
+    {
+      CloseHandle(tidH);
+    }
   }
+#endif
 
   return 0;
 }
@@ -1427,6 +1587,12 @@ amps_tcp_attempt_reconnect(amps_handle transport, unsigned version)
     version = me->connectionVersion;
   }
   res = amps_tcp_handle_disconnect(me, version);
+  if (res == AMPS_E_OK)
+  {
+    // We don't want non-receive threads detaching
+    amps_tcp_set_thread_key(NULL);
+    res = AMPS_E_RETRY;
+  }
   return res;
 }