pybiolib 1.2.561__tar.gz → 1.2.567__tar.gz

{pybiolib-1.2.561 → pybiolib-1.2.567}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybiolib
-Version: 1.2.561
+Version: 1.2.567
 Summary: BioLib Python Client
 License: MIT
 Keywords: biolib

{pybiolib-1.2.561 → pybiolib-1.2.567}/biolib/compute_node/job_worker/executors/docker_executor.py

@@ -27,7 +27,7 @@ from biolib.compute_node import utils as compute_node_utils
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker.docker_image_cache import DockerImageCache
 from biolib.compute_node.job_worker.executors.docker_types import DockerDiffKind
-from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions, StatusUpdate, MetadataToSaveOutput
+from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions, MetadataToSaveOutput, StatusUpdate
 from biolib.compute_node.job_worker.mappings import Mappings, path_without_first_folder
 from biolib.compute_node.job_worker.utilization_reporter_thread import UtilizationReporterThread
 from biolib.compute_node.job_worker.utils import ComputeProcessException
@@ -201,9 +201,6 @@ class DockerExecutor:
             stream=True,
         )
 
-        logger_no_user_data.debug(f'Running diff for Docker container for {job_uuid}')
-        pre_start_diff = self._container.diff()
-
         logger_no_user_data.debug(f'Starting Docker container for {job_uuid}')
         startup_error_string: Optional[str] = None
         try:
@@ -218,7 +215,6 @@ class DockerExecutor:
         self._metadata_for_save_output_on_cancel = MetadataToSaveOutput(
             arguments=module_input['arguments'],
             startup_error_string=startup_error_string,
-            pre_start_diff=pre_start_diff,
         )
 
         if self._options['job']['app_version'].get('stdout_render_type') != 'markdown':
@@ -264,7 +260,6 @@ class DockerExecutor:
             module_output_path=self._options['module_output_path'],
             stderr=full_stderr,
             stdout=full_stdout,
-            pre_start_diff=metadata['pre_start_diff'],
         )
 
     def cleanup(self):
@@ -532,12 +527,11 @@ class DockerExecutor:
         module_output_path: str,
         stderr: bytes,
         stdout: bytes,
-        pre_start_diff: List[Dict],
     ) -> None:
         mapped_files: List[FileInContainer] = []
         try:
             mappings = Mappings(mappings_list=self._options['module']['output_files_mappings'], arguments=arguments)
-            changed_files: List[FileInContainer] = self._get_changed_files_in_docker_container(pre_start_diff)
+            changed_files: List[FileInContainer] = self._get_changed_files_in_docker_container()
 
             for file in changed_files:
                 if file.is_file():
@@ -602,16 +596,35 @@ class DockerExecutor:
 
         return None
 
-    def _get_changed_files_in_docker_container(self, pre_start_diff: List[Dict]) -> List[FileInContainer]:
+    def _get_changed_files_in_docker_container(self) -> List[FileInContainer]:
         from_mappings = [mapping['from_path'] for mapping in self._options['module']['output_files_mappings']]
-        pre_start_diff_paths = [obj['Path'] for obj in pre_start_diff]
-        post_run_diff = self._container.diff()
-        run_diff_paths: List[str] = [
-            obj['Path']
-            for obj in post_run_diff
-            if obj['Kind'] in (DockerDiffKind.CHANGED.value, DockerDiffKind.ADDED.value)
-            and obj['Path'] not in pre_start_diff_paths
-        ]
+        overlay_upper_dir_path = self._get_container_upper_dir_path()
+
+        if not overlay_upper_dir_path:
+            logger_no_user_data.debug(
+                'Docker UpperDir not available. Falling back to container.get_archive() for file extraction'
+            )
+            post_run_diff = self._container.diff()
+            run_diff_paths: List[str] = [
+                obj['Path']
+                for obj in post_run_diff
+                if obj['Kind'] in (DockerDiffKind.CHANGED.value, DockerDiffKind.ADDED.value)
+            ]
+        else:
+            logger_no_user_data.debug(f'overlay_upper_dir_path={overlay_upper_dir_path}')
+            # Recursively find all files in overlay_upper_dir_path
+            run_diff_paths = []
+            for root, _, files in os.walk(overlay_upper_dir_path):
+                # Convert absolute paths to container paths
+                rel_path = os.path.relpath(root, overlay_upper_dir_path)
+                if rel_path == '.':
+                    # Handle the root directory case
+                    for file in files:
+                        run_diff_paths.append(f'/{file}')
+                else:
+                    # Handle subdirectories
+                    for file in files:
+                        run_diff_paths.append(f'/{rel_path}/{file}')
 
         known_directories = set()
         for path in run_diff_paths:
@@ -629,14 +642,6 @@ class DockerExecutor:
                     return True
             return False
 
-        overlay_upper_dir_path = self._get_container_upper_dir_path()
-        if not overlay_upper_dir_path:
-            logger_no_user_data.debug(
-                'Docker UpperDir not available. Falling back to container.get_archive() for file extraction'
-            )
-
-        logger_no_user_data.debug(f'overlay_upper_dir_path={overlay_upper_dir_path}')
-
         files_and_empty_dirs: List[FileInContainer] = []
         for path in run_diff_paths:
             if path not in known_directories and path_is_included_in_from_mappings(path):

{pybiolib-1.2.561 → pybiolib-1.2.567}/biolib/compute_node/job_worker/executors/types.py

@@ -46,4 +46,3 @@ class LocalExecutorOptions(TypedDict):
 class MetadataToSaveOutput(TypedDict):
     arguments: List[str]
     startup_error_string: Optional[str]
-    pre_start_diff: List[Dict]

{pybiolib-1.2.561 → pybiolib-1.2.567}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.2.561"
+version = "1.2.567"
 description = "BioLib Python Client"
 readme = "PYPI_README.md"
 license = "MIT"