pybiolib 1.2.52.dev1__py3-none-any.whl → 1.2.63.dev1__py3-none-any.whl

biolib/_data_record/data_record.py

@@ -6,19 +6,21 @@ from pathlib import Path
 from struct import Struct
 from typing import Callable, Dict, List, Optional, Union, cast
 
-from biolib import api, utils
+from biolib import api
 from biolib._internal import types
 from biolib._internal.data_record import get_data_record_state_from_uri
 from biolib._internal.data_record.data_record import validate_sqlite_v1
+from biolib._internal.data_record.push_data import (
+    push_data_path,
+    validate_data_path_and_get_files_and_size_of_directory,
+)
 from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
-from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib._internal.http_client import HttpClient
 from biolib.api import client as api_client
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersion, DataRecordVersionInfo
 from biolib.biolib_binary_format import LazyLoadedFile
 from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
-from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
 from biolib.utils.app_uri import parse_app_uri
 from biolib.utils.zip.remote_zip import RemoteZip
@@ -85,18 +87,8 @@ class DataRecord:
         self.download_files(output_dir=output_dir, path_filter=path_filter)
 
     def update(self, data_path: str, chunk_size_in_mb: Optional[int] = None) -> None:
-        assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
         BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Data Record')
-
-        if os.path.realpath(data_path) == '/':
-            raise BioLibError('Pushing your root directory is not possible')
-
-        original_working_dir = os.getcwd()
-        os.chdir(data_path)
-        files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
-
-        if data_size_in_bytes > 4_500_000_000_000:
-            raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+        files_to_zip, data_size_in_bytes = validate_data_path_and_get_files_and_size_of_directory(data_path)
 
         # validate data record
         detailed_dict: types.DataRecordDetailedDict = self._get_detailed_dict()
@@ -114,43 +106,20 @@ class DataRecord:
                 else:
                     raise Exception(f"Error processing data record validation: unknown rule type {rule['type']}")
 
-        min_chunk_size_bytes = 10_000_000
-        chunk_size_in_bytes: int
-        if chunk_size_in_mb:
-            chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
-            if chunk_size_in_bytes < min_chunk_size_bytes:
-                logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
-                chunk_size_in_bytes = min_chunk_size_bytes
-        else:
-            # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
-            chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
-
-        data_size_in_mb = round(data_size_in_bytes / 10**6)
-        logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
-
         response = api.client.post(path='/lfs/versions/', data={'resource_uuid': self._state['resource_uuid']})
         data_record_version: DataRecordVersion = response.json()
-        iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
-
-        multipart_uploader = utils.MultiPartUploader(
-            use_process_pool=True,
-            get_presigned_upload_url_request=dict(
-                headers=None,
-                requires_biolib_auth=True,
-                path=f"/lfs/versions/{data_record_version['uuid']}/presigned_upload_url/",
-            ),
-            complete_upload_request=dict(
-                headers=None,
-                requires_biolib_auth=True,
-                path=f"/lfs/versions/{data_record_version['uuid']}/complete_upload/",
-            ),
+        resource_version_uuid = data_record_version['uuid']
+
+        push_data_path(
+            data_path=data_path,
+            data_size_in_bytes=data_size_in_bytes,
+            files_to_zip=files_to_zip,
+            resource_version_uuid=resource_version_uuid,
+            chunk_size_in_mb=chunk_size_in_mb,
         )
 
-        multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
-        os.chdir(original_working_dir)
-
         api.client.patch(
-            path=f"/resources/versions/{data_record_version['uuid']}/",
+            path=f'/resources/versions/{resource_version_uuid}/',
             data={'state': 'published', 'set_as_active': True},
         )
 

biolib/_internal/data_record/push_data.py

@@ -0,0 +1,67 @@
+import os
+
+from biolib import utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
+from biolib._internal.types.typing import List, Optional, Tuple
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger
+
+
+def validate_data_path_and_get_files_and_size_of_directory(data_path: str) -> Tuple[List[str], int]:
+    assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
+
+    if os.path.realpath(data_path) == '/':
+        raise BioLibError('Pushing your root directory is not possible')
+
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+    files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
+    os.chdir(original_working_dir)
+
+    if data_size_in_bytes > 4_500_000_000_000:
+        raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+
+    return files_to_zip, data_size_in_bytes
+
+
+def push_data_path(
+    data_path: str,
+    data_size_in_bytes: int,
+    files_to_zip: List[str],
+    resource_version_uuid: str,
+    chunk_size_in_mb: Optional[int] = None,
+) -> None:
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+
+    min_chunk_size_bytes = 10_000_000
+    chunk_size_in_bytes: int
+    if chunk_size_in_mb:
+        chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
+        if chunk_size_in_bytes < min_chunk_size_bytes:
+            logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
+            chunk_size_in_bytes = min_chunk_size_bytes
+    else:
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
+
+    data_size_in_mb = round(data_size_in_bytes / 10**6)
+    logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
+
+    iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
+    multipart_uploader = utils.MultiPartUploader(
+        use_process_pool=True,
+        get_presigned_upload_url_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f'/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
+        ),
+        complete_upload_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f'/lfs/versions/{resource_version_uuid}/complete_upload/',
+        ),
+    )
+
+    multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
+    os.chdir(original_working_dir)

biolib/_internal/push_application.py

@@ -6,6 +6,10 @@ import rich.progress
 import yaml
 
 from biolib import api, utils
+from biolib._internal.data_record.push_data import (
+    push_data_path,
+    validate_data_path_and_get_files_and_size_of_directory,
+)
 from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
@@ -94,10 +98,25 @@ def push_application(
     zip_filters.add('.biolib/config.yml')
 
     input_files_maps_to_root = False
+    app_data_path: Optional[Path] = None
     try:
         with open(config_yml_path) as config_yml_file:
             config = yaml.safe_load(config_yml_file.read())
 
+        app_data = config.get('app_data')
+        if app_data:
+            if not isinstance(app_data, str):
+                raise BioLibError(
+                    f'In .biolib/config.yml the value of "app_data" must be a string but got {type(app_data)}'
+                )
+
+            app_data_path = app_path_absolute.joinpath(app_data).resolve()
+            if not app_data_path.is_dir():
+                raise BioLibError(
+                    'In .biolib/config.yml the value of "app_data" must be a path to a directory '
+                    'in the application directory'
+                )
+
         license_file_relative_path = config.get('license_file', 'LICENSE')
         if app_path_absolute.joinpath(license_file_relative_path).is_file():
             zip_filters.add(license_file_relative_path)
@@ -118,11 +137,14 @@ def push_application(
             if module.get('source_files'):
                 zip_filters.add('*')
 
-            for mapping in module['input_files']:
+            for mapping in module.get('input_files', []):
                 mapping_parts = mapping.split(' ')
                 if len(mapping_parts) == 3 and mapping_parts[2] == '/':
                     input_files_maps_to_root = True
 
+    except BioLibError as error:
+        raise error from None
+
     except Exception as error:
         raise BioLibError('Failed to parse the .biolib/config.yml file') from error
 
@@ -175,6 +197,17 @@ def push_application(
         else None,
     )
 
+    if app_data_path:
+        app_data_files_to_zip, app_data_size_in_bytes = validate_data_path_and_get_files_and_size_of_directory(
+            data_path=str(app_data_path),
+        )
+        push_data_path(
+            resource_version_uuid=new_app_version_json['public_id'],
+            data_path=str(app_data_path),
+            data_size_in_bytes=app_data_size_in_bytes,
+            files_to_zip=app_data_files_to_zip,
+        )
+
     #  Don't push docker images if copying from another app version
     docker_tags = new_app_version_json.get('docker_tags', {})
     if not app_version_to_copy_images_from and docker_tags:

{pybiolib-1.2.52.dev1.dist-info → pybiolib-1.2.63.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.2.52.dev1
+Version: 1.2.63.dev1
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.2.52.dev1.dist-info → pybiolib-1.2.63.dev1.dist-info}/RECORD

@@ -1,10 +1,11 @@
 LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
 PYPI_README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
 biolib/__init__.py,sha256=q_YhAYw51Vq16IKtSk8_MJclDAa4CfCmPhvWDYmrSIg,4393
-biolib/_data_record/data_record.py,sha256=-_EAcyheLWIkx02vQYygzEjxArYqx9XuQQYeE5GtuzU,12672
+biolib/_data_record/data_record.py,sha256=zVAhFU1RLI1-ptoQ_l639RNwrMANXV9j75yXHvB7dtA,10950
 biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/_internal/data_record/__init__.py,sha256=fGdME6JGRU_2VxpJbYpGXYndjN-feUkmKY4fuMyq3cg,76
 biolib/_internal/data_record/data_record.py,sha256=g_-jdy5-Zem3dthwxJj2OuQqkDGTyc-iGqN1rtYYD1A,4418
+biolib/_internal/data_record/push_data.py,sha256=-L3a_7zZzDCXabBu3O4lWPMAMeBbeRPTrBlEM-_5SCI,2693
 biolib/_internal/data_record/remote_storage_endpoint.py,sha256=eCptuZ4DMAPnaNCVDvpWXwXGI6Jac9U1N5dqU8Cj95Q,1732
 biolib/_internal/file_utils.py,sha256=4jT6j7bB21c0JNn5BfnyWQib_zt0CVtJ_TiOFOStRcE,2604
 biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
@@ -14,7 +15,7 @@ biolib/_internal/lfs/__init__.py,sha256=gSWo_xg61UniYgD7yNYxeT4I9uaXBCBSi3_nmZjn
 biolib/_internal/lfs/cache.py,sha256=pQS2np21rdJ6I3DpoOutnzPHpLOZgUIS8TMltUJk_k4,2226
 biolib/_internal/libs/__init__.py,sha256=Jdf4tNPqe_oIIf6zYml6TiqhL_02Vyqwge6IELrAFhw,98
 biolib/_internal/libs/fusepy/__init__.py,sha256=AWDzNFS-XV_5yKb0Qx7kggIhPzq1nj_BZS5y2Nso08k,41944
-biolib/_internal/push_application.py,sha256=b1WvlijJYOLp7f43AoAzjINKkALM9DVB8yfSetEP4Z4,10381
+biolib/_internal/push_application.py,sha256=mKs3kIKW-ZYfz3Cy6LIyFBwsWkbcGZ9zgMk-xn5NDyg,11660
 biolib/_internal/runtime.py,sha256=BiHl4klUHr36MCpqKaUso4idHeBZfPAahLYRQrabFqA,486
 biolib/_internal/types/__init__.py,sha256=xLgOQJFh3GRtiqIJq7MaqHReZx4pp34_zcaFQ_JjuJ4,198
 biolib/_internal/types/app.py,sha256=Mz2QGD_jESX-K9JYnLWPo4YA__Q_1FQQTk9pvidCohU,118
@@ -118,8 +119,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=ZQFcaE37B2dtucN2zDjOmdya_X0ITc1zBFZJNQY13XA,5183
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.52.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.52.dev1.dist-info/METADATA,sha256=8m8D3rQE-Z7a3nLw8V5ky0Dt4K0bHefBONkoUaiieoo,1511
-pybiolib-1.2.52.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.2.52.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.52.dev1.dist-info/RECORD,,
+pybiolib-1.2.63.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.63.dev1.dist-info/METADATA,sha256=t8YhPOU4slbJKvi3kieokCcqoKqgyqaYf5Tjwi7Vs3s,1511
+pybiolib-1.2.63.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.2.63.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.63.dev1.dist-info/RECORD,,