pybiolib 1.1.2145__py3-none-any.whl → 1.1.2155__py3-none-any.whl

biolib/_data_record/data_record.py

@@ -5,21 +5,24 @@ from fnmatch import fnmatch
 from struct import Struct
 from typing import Callable, Dict, List, Union, cast
 
-from biolib import api
+from biolib import api, utils
 from biolib._internal import types
-from biolib._internal.data_record import get_data_record_state_from_uri, push_data_record_version
+from biolib._internal.data_record import get_data_record_state_from_uri
 from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib._internal.http_client import HttpClient
 from biolib.api import client as api_client
 from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersionInfo
+from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersion, DataRecordVersionInfo
 from biolib.biolib_binary_format import LazyLoadedFile
 from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
+from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
 from biolib.typing_utils import Optional as _Optional
 from biolib.utils.app_uri import parse_app_uri
 from biolib.utils.zip.remote_zip import RemoteZip
 
+
 PathFilter = Union[str, Callable[[str], bool]]
 
 
@@ -83,10 +86,63 @@ class DataRecord:
 
     def update(self, data_path: str, chunk_size_in_mb: _Optional[int] = None) -> None:
         assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
-        uri = push_data_record_version(
-            data_record_uuid=self._state['resource_uuid'], input_dir=data_path, chunk_size_in_mb=chunk_size_in_mb
+        BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Data Record')
+
+        if os.path.realpath(data_path) == '/':
+            raise BioLibError('Pushing your root directory is not possible')
+
+        original_working_dir = os.getcwd()
+        os.chdir(data_path)
+        files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
+
+        if data_size_in_bytes > 4_500_000_000_000:
+            raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+
+        # validate data record
+        detailed_dict: types.DataRecordDetailedDict = self._get_detailed_dict()
+        if detailed_dict['type']:
+            # only validate if data record has a type
+            data_record_type: types.DataRecordTypeDict = detailed_dict['type']
+            logger.info(f"Validating data record of type {data_record_type['name']}")
+            for rule in data_record_type['validation_rules']:
+                logger.info(f"Validating rule {rule['type']} for {rule['path']}...")
+
+        min_chunk_size_bytes = 10_000_000
+        chunk_size_in_bytes: int
+        if chunk_size_in_mb:
+            chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
+            if chunk_size_in_bytes < min_chunk_size_bytes:
+                logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
+                chunk_size_in_bytes = min_chunk_size_bytes
+        else:
+            # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+            chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
+
+        data_size_in_mb = round(data_size_in_bytes / 10**6)
+        logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
+
+        response = api.client.post(path='/lfs/versions/', data={'resource_uuid': self._state['resource_uuid']})
+        data_record_version: DataRecordVersion = response.json()
+        iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
+
+        multipart_uploader = utils.MultiPartUploader(
+            use_process_pool=True,
+            get_presigned_upload_url_request=dict(
+                headers=None,
+                requires_biolib_auth=True,
+                path=f"/lfs/versions/{data_record_version['uuid']}/presigned_upload_url/",
+            ),
+            complete_upload_request=dict(
+                headers=None,
+                requires_biolib_auth=True,
+                path=f"/lfs/versions/{data_record_version['uuid']}/complete_upload/",
+            ),
         )
-        self._state = get_data_record_state_from_uri(uri)
+
+        multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
+        os.chdir(original_working_dir)
+        logger.info(f"Successfully pushed a new Data Record version '{data_record_version['uri']}'")
+        self._state = get_data_record_state_from_uri(data_record_version['uri'])
 
     @staticmethod
     def get_by_uri(uri: str) -> 'DataRecord':
@@ -112,14 +168,15 @@ class DataRecord:
                 'name': uri_parsed['app_name'],
             },
         )
-        data_record: DataRecordInfo = response.json()
-        logger.info(f"Successfully created new Data Record '{data_record['uri']}'")
+        data_record_info: DataRecordInfo = response.json()
+        logger.info(f"Successfully created new Data Record '{data_record_info['uri']}'")
 
         if data_path is not None:
-            record_version_uri = push_data_record_version(data_record_uuid=data_record['uuid'], input_dir=data_path)
-            return DataRecord.get_by_uri(uri=record_version_uri)
+            data_record = DataRecord.get_by_uri(uri=data_record_info['uri'])
+            data_record.update(data_path=data_path)
+            return data_record
         else:
-            return DataRecord.get_by_uri(uri=data_record_uri)
+            return DataRecord.get_by_uri(uri=data_record_info['uri'])
 
     @staticmethod
     def fetch(uri: _Optional[str] = None, count: _Optional[int] = None) -> List['DataRecord']:

biolib/_internal/data_record/__init__.py

@@ -1 +1 @@
-from .data_record import get_data_record_state_from_uri, push_data_record_version
+from .data_record import get_data_record_state_from_uri

biolib/_internal/data_record/data_record.py

@@ -1,67 +1,6 @@
-import os
-from typing import Optional
-
-from biolib import api, utils
-from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.api import client as api_client
-from biolib.biolib_api_client import AppGetResponse, BiolibApiClient
-from biolib.biolib_api_client.lfs_types import DataRecordVersion, DataRecordVersionInfo
-from biolib.biolib_errors import BioLibError
-from biolib.biolib_logging import logger
-
-
-def push_data_record_version(data_record_uuid: str, input_dir: str, chunk_size_in_mb: Optional[int] = None) -> str:
-    BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Data Record')
-
-    if not os.path.isdir(input_dir):
-        raise BioLibError(f'Could not find folder at {input_dir}')
-
-    if os.path.realpath(input_dir) == '/':
-        raise BioLibError('Pushing your root directory is not possible')
-
-    original_working_dir = os.getcwd()
-    os.chdir(input_dir)
-    files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
-
-    if data_size_in_bytes > 4_500_000_000_000:
-        raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
-
-    min_chunk_size_bytes = 10_000_000
-    chunk_size_in_bytes: int
-    if chunk_size_in_mb:
-        chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
-        if chunk_size_in_bytes < min_chunk_size_bytes:
-            logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
-            chunk_size_in_bytes = min_chunk_size_bytes
-    else:
-        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
-        chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
-
-    data_size_in_mb = round(data_size_in_bytes / 10**6)
-    print(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
-
-    response = api.client.post(path='/lfs/versions/', data={'resource_uuid': data_record_uuid})
-    data_record_version: DataRecordVersion = response.json()
-    iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
-
-    multipart_uploader = utils.MultiPartUploader(
-        use_process_pool=True,
-        get_presigned_upload_url_request=dict(
-            headers=None,
-            requires_biolib_auth=True,
-            path=f"/lfs/versions/{data_record_version['uuid']}/presigned_upload_url/",
-        ),
-        complete_upload_request=dict(
-            headers=None,
-            requires_biolib_auth=True,
-            path=f"/lfs/versions/{data_record_version['uuid']}/complete_upload/",
-        ),
-    )
-
-    multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
-    os.chdir(original_working_dir)
-    logger.info(f"Successfully pushed a new Data Record version '{data_record_version['uri']}'")
-    return data_record_version['uri']
+from biolib.biolib_api_client import AppGetResponse
+from biolib.biolib_api_client.lfs_types import DataRecordVersionInfo
 
 
 def get_data_record_state_from_uri(uri) -> 'DataRecordVersionInfo':

{pybiolib-1.1.2145.dist-info → pybiolib-1.1.2155.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.2145
+Version: 1.1.2155
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.2145.dist-info → pybiolib-1.1.2155.dist-info}/RECORD

@@ -1,10 +1,10 @@
 LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
 README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
 biolib/__init__.py,sha256=_tThyzISH81yS9KXP_X3qEiKXmsIp5XOBcJIODfLVnc,4338
-biolib/_data_record/data_record.py,sha256=KTHrVy_wj6FU3ckD0_EsakyNJcniLIdT9c3ugujNRoI,9210
+biolib/_data_record/data_record.py,sha256=Sud8yXz7yR6YW4V6OqE7nO6I4a0TdqijmMTZwwU59j8,12152
 biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-biolib/_internal/data_record/__init__.py,sha256=0T0CV6PfKc8itjMu-48sCJjcZQEzXl1ZLBqG_LjJTqQ,82
-biolib/_internal/data_record/data_record.py,sha256=D0BaC8WhnkM564eKUI69hVHUkKY1In0cyfpjxYyWk18,3363
+biolib/_internal/data_record/__init__.py,sha256=wLOy3Pb7dWYik5eQtQM00DH2AWC-M5RbTjwh9InPiqo,56
+biolib/_internal/data_record/data_record.py,sha256=If4SQj-XwKSPzCpaWA01LEGKalZ6DEjD5PJZRtl3Mao,556
 biolib/_internal/data_record/remote_storage_endpoint.py,sha256=eCptuZ4DMAPnaNCVDvpWXwXGI6Jac9U1N5dqU8Cj95Q,1732
 biolib/_internal/file_utils.py,sha256=4jT6j7bB21c0JNn5BfnyWQib_zt0CVtJ_TiOFOStRcE,2604
 biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
@@ -116,8 +116,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=jC5WhH63FTD7SLFJbxQGA2hOt9NTwq9zHl_BEec1Z0c,4907
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.2145.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.2145.dist-info/METADATA,sha256=rpVBXD5_Q8O8-SBz_G9jwhvCWRz_ww1DsPc0eApjBMc,1508
-pybiolib-1.1.2145.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.2145.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.2145.dist-info/RECORD,,
+pybiolib-1.1.2155.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.2155.dist-info/METADATA,sha256=tu73TYRGUjbJ3MfO15gYq2UO81SYCy508-wwmoHN3WQ,1508
+pybiolib-1.1.2155.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.1.2155.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.2155.dist-info/RECORD,,