pybiolib 1.2.346.dev1__py3-none-any.whl → 1.2.366__py3-none-any.whl

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -44,8 +44,10 @@ class DockerExecutor:
         self._send_system_exception = options['send_system_exception']
         self._send_stdout_and_stderr = options['send_stdout_and_stderr']
         self._random_docker_id = compute_node_utils.random_string(15)
-        self._reserved_system_ram = int(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') * 0.06) + 500_000_000
-        logger_no_user_data.info(f"System RAM reserved: {self._reserved_system_ram}")
+        total_memory_in_bytes = int(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES'))
+        system_reserved_memory = int(total_memory_in_bytes * 0.1) + 500_000_000
+        self._available_memory_in_bytes = total_memory_in_bytes - system_reserved_memory
+        logger_no_user_data.info(f'Available memory for containers: {self._available_memory_in_bytes} bytes')
 
         if utils.IS_RUNNING_IN_CLOUD:
             self._compute_process_dir = os.getenv('BIOLIB_USER_DATA_PATH')
@@ -164,7 +166,6 @@ class DockerExecutor:
             may_contain_user_data=False,
         )
 
-
     def _execute_helper(self, module_input) -> None:
         job_uuid = self._options['job']['public_id']
         logger_no_user_data.debug(f'Initializing container for {job_uuid}.')
@@ -250,7 +251,6 @@ class DockerExecutor:
         logger_no_user_data.debug(f'Docker container exited with code {exit_code} for {job_uuid}')
         self._save_module_output_from_container(exit_code, self._metadata_for_save_output_on_cancel)
 
-
     def _save_module_output_from_container(self, exit_code: int, metadata: MetadataToSaveOutput) -> None:
         full_stdout = self._docker_api_client.logs(self._container.id, stdout=True, stderr=False)
         full_stderr = self._docker_api_client.logs(self._container.id, stdout=False, stderr=True)
@@ -267,7 +267,6 @@ class DockerExecutor:
             pre_start_diff=metadata['pre_start_diff'],
         )
 
-
     def cleanup(self):
         # Don't clean up if already in the process of doing so, or done doing so
         if self._is_cleaning_up:
@@ -283,7 +282,7 @@ class DockerExecutor:
                 logger_no_user_data.debug(f'Container state {self._docker_container.status}')
                 self._save_module_output_from_container(
                     exit_code=self._docker_container.attrs['State']['ExitCode'],
-                    metadata=self._metadata_for_save_output_on_cancel
+                    metadata=self._metadata_for_save_output_on_cancel,
                 )
                 logger_no_user_data.debug('Saved results')
             except BaseException as error:
@@ -412,8 +411,15 @@ class DockerExecutor:
 
             if utils.IS_RUNNING_IN_CLOUD:
                 cloud_job = self._options['cloud_job']
-                create_container_args['mem_limit'] = (
-                    f'{cloud_job["reserved_memory_in_bytes"] - self._reserved_system_ram}b'
+                container_memory_limit_in_bytes = min(
+                    cloud_job['reserved_memory_in_bytes'], self._available_memory_in_bytes
+                )
+                create_container_args['mem_limit'] = f'{container_memory_limit_in_bytes}b'
+                logger_no_user_data.debug(
+                    'Setting container memory limit to '
+                    f'{container_memory_limit_in_bytes} bytes '
+                    f'(requested: {cloud_job["reserved_memory_in_bytes"]}, '
+                    f'available: {self._available_memory_in_bytes})'
                 )
                 create_container_args['nano_cpus'] = cloud_job['reserved_cpu_in_nano_shares']
                 create_container_args['pids_limit'] = 10_000

{pybiolib-1.2.346.dev1.dist-info → pybiolib-1.2.366.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybiolib
-Version: 1.2.346.dev1
+Version: 1.2.366
 Summary: BioLib Python Client
 License: MIT
 Keywords: biolib

{pybiolib-1.2.346.dev1.dist-info → pybiolib-1.2.366.dist-info}/RECORD

@@ -76,7 +76,7 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=NpDSgNWbnC0Qr6aFaQMEfhib3O9BMOPsia-YqeTMhpU,30966
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=UxWUSk15mM_H73q2k5_kBB1IWGYGphyXbyC-Bm0QneU,31550
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=gw_T-oq0lty8Hg8WsTrqhtV-kyOF_l5nr9Xe8-kzetc,1657
@@ -118,8 +118,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=Ozk0blGtPur_D9MwShD02r_mphyQmgZkx-lOHOwnlIM,6730
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.346.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.346.dev1.dist-info/METADATA,sha256=E1vqmPmdZsfLHwqPzcZQhdS-GBg9M2hl1eBI44rKi80,1575
-pybiolib-1.2.346.dev1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-pybiolib-1.2.346.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.346.dev1.dist-info/RECORD,,
+pybiolib-1.2.366.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.366.dist-info/METADATA,sha256=0VkWxDelNbXapivceuFPSl1TiSZa1myr6plhWi7HNRQ,1570
+pybiolib-1.2.366.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+pybiolib-1.2.366.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.366.dist-info/RECORD,,