pybiolib 1.1.2208__py3-none-any.whl → 1.1.2218__py3-none-any.whl

biolib/compute_node/webserver/worker_thread.py

@@ -2,23 +2,23 @@ import base64
 import os
 import random
 import shutil
+import socket
 import sys
-import time
 import threading
-import socket
+import time
 from queue import Queue
 
-from biolib import utils
+from biolib import api, utils
+from biolib.biolib_binary_format import ModuleOutputV2, SystemException, SystemStatusUpdate
 from biolib.biolib_binary_format.utils import LocalFileIndexableBuffer
+from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker import JobWorkerProcess
 from biolib.compute_node.job_worker.job_storage import JobStorage
 from biolib.compute_node.socker_listener_thread import SocketListenerThread
 from biolib.compute_node.socket_sender_thread import SocketSenderThread
+from biolib.compute_node.utils import SystemExceptionCodes, WorkerThreadException, get_package_type
 from biolib.compute_node.webserver import webserver_utils
-from biolib.biolib_binary_format import SystemStatusUpdate, SystemException, ModuleOutputV2
-from biolib.compute_node.utils import get_package_type, WorkerThreadException, SystemExceptionCodes
-from biolib.biolib_logging import logger, logger_no_user_data
 
 SOCKET_HOST = '127.0.0.1'
 
@@ -37,7 +37,7 @@ class WorkerThread(threading.Thread):
             self._sender_thread = None
             self._start_and_connect_to_compute_process()
 
-            logger.debug(f"WorkerThread connected to port {self._socket_port}")
+            logger.debug(f'WorkerThread connected to port {self._socket_port}')
 
         except Exception as exception:
             logger_no_user_data.error(exception)
@@ -79,20 +79,16 @@ class WorkerThread(threading.Thread):
                     if progress == 94:
                         # Get Job exit code
                         try:
-                            module_output_path = os.path.join(self._job_temporary_dir,
-                                                              JobStorage.module_output_file_name)
-                            module_output = ModuleOutputV2(
-                                buffer=LocalFileIndexableBuffer(
-                                    filename=module_output_path
-                                )
+                            module_output_path = os.path.join(
+                                self._job_temporary_dir,
+                                JobStorage.module_output_file_name,
                             )
+                            module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
                             self.compute_state['exit_code'] = module_output.get_exit_code()
                             logger_no_user_data.debug(f"Got exit code: {self.compute_state['exit_code']}")
 
                         except Exception as error:  # pylint: disable=broad-except
-                            logger_no_user_data.error(
-                                f'Could not get exit_code from module output due to: {error}'
-                            )
+                            logger_no_user_data.error(f'Could not get exit_code from module output due to: {error}')
 
                         if utils.IS_RUNNING_IN_CLOUD:
                             JobStorage.upload_module_output(
@@ -107,7 +103,7 @@ class WorkerThread(threading.Thread):
                 elif package_type == 'SystemException':
                     error_code = SystemException(package).deserialize()
                     self.compute_state['status']['error_code'] = error_code
-                    logger.debug("Hit error. Terminating Worker Thread and Compute Process")
+                    logger.debug('Hit error. Terminating Worker Thread and Compute Process')
                     self.compute_state['progress'] = 95
                     self.terminate()
 
@@ -153,10 +149,10 @@ class WorkerThread(threading.Thread):
 
         # Starting a thread for accepting connections before starting the process that should to connect to the socket
         logger_no_user_data.debug('Starting connection thread')
-        self._connection_thread = threading.Thread(target=self._accept_new_socket_connection, args=[
-            received_messages_queue,
-            messages_to_send_queue
-        ])
+        self._connection_thread = threading.Thread(
+            target=self._accept_new_socket_connection,
+            args=[received_messages_queue, messages_to_send_queue],
+        )
         self._connection_thread.start()
         logger_no_user_data.debug('Started connection thread')
         logger_no_user_data.debug('Starting compute process')
@@ -177,6 +173,16 @@ class WorkerThread(threading.Thread):
         self._sender_thread.start()
 
     def terminate(self) -> None:
+        cloud_job_uuid = self.compute_state['cloud_job_id']
+        exit_code = self.compute_state.get('exit_code')
+        system_exception_code = self.compute_state['status'].get('error_code')
+        if utils.IS_RUNNING_IN_CLOUD:
+            CloudUtils.finish_cloud_job(
+                cloud_job_id=cloud_job_uuid,
+                system_exception_code=system_exception_code,
+                exit_code=exit_code,
+            )
+
         deregistered_due_to_error = False
         if self._job_worker_process:
             logger_no_user_data.debug(
@@ -184,7 +190,8 @@ class WorkerThread(threading.Thread):
             )
             self._job_worker_process.terminate()
 
-            for _ in range(10):
+            clean_up_timeout_in_seconds = 600
+            for _ in range(clean_up_timeout_in_seconds):
                 if self._job_worker_process.exitcode is not None:
                     logger_no_user_data.debug(
                         f'Job "{self._job_uuid}" worker process exitcode {self._job_worker_process.exitcode}'
@@ -196,35 +203,25 @@ class WorkerThread(threading.Thread):
 
             if self._job_worker_process.exitcode is None:
                 # TODO: Figure out if more error handling is necessary here
-                logger_no_user_data.error(f'Job {self._job_uuid} worker process did not exit within 10 seconds')
+                logger_no_user_data.error(
+                    f'Job {self._job_uuid} worker process did not exit within {clean_up_timeout_in_seconds} seconds'
+                )
                 if utils.IS_RUNNING_IN_CLOUD:
                     logger_no_user_data.error('Deregistering compute node...')
                     CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
         # Delete result as error occurred
-        system_exception_code = self.compute_state['status'].get('error_code')
         if system_exception_code and os.path.exists(self._job_temporary_dir):
             shutil.rmtree(self._job_temporary_dir)
 
-        exit_code = self.compute_state.get('exit_code')
-
-        if utils.IS_RUNNING_IN_CLOUD:
-            # Get and send compute node exception code and job exit code if present
-            logger_no_user_data.debug(f"Sending exit code {exit_code}")
-            CloudUtils.finish_cloud_job(
-                cloud_job_id=self.compute_state['cloud_job_id'],
-                system_exception_code=system_exception_code,
-                exit_code=exit_code
-            )
-
         if self._socket:
             self._socket.close()
 
         if self._connection:
             self._connection.close()
 
-        if self.compute_state['progress'] == 95:
+        if not utils.IS_RUNNING_IN_CLOUD and self.compute_state['progress'] == 95:
             seconds_to_sleep = 60  # 1 minute
             logger_no_user_data.debug(
                 f'Job "{self._job_uuid}" worker thread sleeping for {seconds_to_sleep} seconds before cleaning up'
@@ -245,12 +242,18 @@ class WorkerThread(threading.Thread):
                 f'Job "{self._job_uuid}" could not be found, maybe it has already been cleaned up'
             )
 
-        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread terminated')
-
         if utils.IS_RUNNING_IN_CLOUD:
+            config = CloudUtils.get_webserver_config()
+            logger_no_user_data.debug(f'Job "{self._job_uuid}" reporting CloudJob "{cloud_job_uuid}" as cleaned up...')
+            api.client.post(
+                path=f'/internal/compute-nodes/cloud-jobs/{cloud_job_uuid}/cleaned-up/',
+                headers={'Compute-Node-Auth-Token': config['compute_node_info']['auth_token']},
+            )
+
             if deregistered_due_to_error:
                 CloudUtils.shutdown()  # shutdown now
             else:
                 webserver_utils.update_auto_shutdown_time()
 
+        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread exiting...')
         sys.exit()

{pybiolib-1.1.2208.dist-info → pybiolib-1.1.2218.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.2208
+Version: 1.1.2218
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.2208.dist-info → pybiolib-1.1.2218.dist-info}/RECORD

@@ -95,7 +95,7 @@ biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLU
 biolib/compute_node/webserver/webserver.py,sha256=15PkRyhtdtSgFDxa0z78aPO4ciZURsFqJYi-HtUmZF8,6494
 biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
 biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
-biolib/compute_node/webserver/worker_thread.py,sha256=26tG73TADnOcXsAr7Iyf6smrLlCqB4x-vvmpUb8WqnA,11569
+biolib/compute_node/webserver/worker_thread.py,sha256=AfzSK6ZXXHB1QufemIEmCbVjvLPcOieNJK4xJRRM7B0,11871
 biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/experiments/experiment.py,sha256=jIRixmQm3Gq9YdJ3I0-rE1vFukXqq6U4zXehFOJ1yZk,7614
 biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
@@ -116,8 +116,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=ZQFcaE37B2dtucN2zDjOmdya_X0ITc1zBFZJNQY13XA,5183
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.2208.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.2208.dist-info/METADATA,sha256=2yI-Ekgvu56nB9gdvkhAlulW1Ww7eh3fgQU8CPaIyJU,1508
-pybiolib-1.1.2208.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.2208.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.2208.dist-info/RECORD,,
+pybiolib-1.1.2218.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.2218.dist-info/METADATA,sha256=syMRSPiPLeFeF4Z5KUmOiQiWBjkohMPb1uARFQZLOK0,1508
+pybiolib-1.1.2218.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.1.2218.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.2218.dist-info/RECORD,,