pybiolib 1.2.1056__py3-none-any.whl → 1.2.1642__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pybiolib might be problematic. Click here for more details.
- biolib/__init__.py +33 -10
- biolib/_data_record/data_record.py +24 -11
- biolib/_index/__init__.py +0 -0
- biolib/_index/index.py +51 -0
- biolib/_index/types.py +7 -0
- biolib/_internal/data_record/data_record.py +1 -1
- biolib/_internal/data_record/push_data.py +1 -1
- biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
- biolib/_internal/file_utils.py +7 -4
- biolib/_internal/index/__init__.py +1 -0
- biolib/_internal/index/index.py +18 -0
- biolib/_internal/lfs/cache.py +4 -2
- biolib/_internal/push_application.py +89 -23
- biolib/_internal/runtime.py +2 -0
- biolib/_internal/templates/gui_template/App.tsx +38 -2
- biolib/_internal/templates/gui_template/Dockerfile +2 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/package.json +1 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
- biolib/_internal/templates/gui_template/vite.config.mts +2 -1
- biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
- biolib/_internal/templates/init_template/Dockerfile +2 -0
- biolib/_internal/utils/__init__.py +25 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_runtime/runtime.py +9 -0
- biolib/_session/session.py +7 -5
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +69 -0
- biolib/_shared/types/resource.py +17 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/{_internal → _shared}/types/resource_permission.py +1 -1
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/client.py +1 -1
- biolib/app/app.py +56 -23
- biolib/biolib_api_client/app_types.py +1 -6
- biolib/biolib_api_client/biolib_app_api.py +17 -0
- biolib/biolib_binary_format/module_input.py +8 -0
- biolib/biolib_binary_format/remote_endpoints.py +3 -3
- biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
- biolib/cli/__init__.py +2 -1
- biolib/cli/data_record.py +17 -0
- biolib/cli/index.py +32 -0
- biolib/cli/lfs.py +1 -1
- biolib/cli/start.py +14 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
- biolib/compute_node/job_worker/executors/docker_types.py +1 -1
- biolib/compute_node/job_worker/executors/types.py +6 -5
- biolib/compute_node/job_worker/job_worker.py +149 -93
- biolib/compute_node/job_worker/large_file_system.py +2 -6
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
- biolib/compute_node/remote_host_proxy.py +125 -67
- biolib/compute_node/utils.py +2 -0
- biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +64 -19
- biolib/experiments/experiment.py +98 -16
- biolib/jobs/job.py +119 -29
- biolib/jobs/job_result.py +70 -33
- biolib/jobs/types.py +1 -0
- biolib/sdk/__init__.py +17 -2
- biolib/typing_utils.py +1 -1
- biolib/utils/cache_state.py +2 -2
- biolib/utils/seq_util.py +1 -1
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/METADATA +4 -2
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/RECORD +84 -66
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/WHEEL +1 -1
- biolib/_internal/types/__init__.py +0 -6
- biolib/utils/app_uri.py +0 -57
- /biolib/{_internal → _shared}/types/account.py +0 -0
- /biolib/{_internal → _shared}/types/account_member.py +0 -0
- /biolib/{_internal → _shared}/types/app.py +0 -0
- /biolib/{_internal → _shared}/types/data_record.py +0 -0
- /biolib/{_internal → _shared}/types/experiment.py +0 -0
- /biolib/{_internal → _shared}/types/file_node.py +0 -0
- /biolib/{_internal → _shared}/types/push.py +0 -0
- /biolib/{_internal/types/resource.py → _shared/types/resource_types.py} +0 -0
- /biolib/{_internal → _shared}/types/resource_version.py +0 -0
- /biolib/{_internal → _shared}/types/result.py +0 -0
- /biolib/{_internal → _shared}/types/typing.py +0 -0
- /biolib/{_internal → _shared}/types/user.py +0 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/entry_points.txt +0 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info/licenses}/LICENSE +0 -0
biolib/__init__.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# ruff: noqa: I001
|
|
1
2
|
# Imports to hide
|
|
2
3
|
import os
|
|
3
4
|
from urllib.parse import urlparse as _urlparse
|
|
@@ -15,6 +16,7 @@ from biolib.jobs.job import Result as _Result
|
|
|
15
16
|
from biolib import user as _user
|
|
16
17
|
from biolib.typing_utils import List, Optional, cast as _cast
|
|
17
18
|
from biolib._data_record.data_record import DataRecord as _DataRecord
|
|
19
|
+
from biolib._internal.utils.job_url import parse_result_id_or_url as _parse_result_id_or_url
|
|
18
20
|
|
|
19
21
|
import biolib.api
|
|
20
22
|
import biolib.app
|
|
@@ -22,7 +24,6 @@ import biolib.cli
|
|
|
22
24
|
import biolib.sdk
|
|
23
25
|
import biolib.utils
|
|
24
26
|
|
|
25
|
-
|
|
26
27
|
# ------------------------------------ Function definitions for public Python API ------------------------------------
|
|
27
28
|
|
|
28
29
|
|
|
@@ -83,43 +84,65 @@ def search(
|
|
|
83
84
|
|
|
84
85
|
|
|
85
86
|
def get_job(job_id: str, job_token: Optional[str] = None) -> _Result:
|
|
86
|
-
r"""Get a job by its ID.
|
|
87
|
+
r"""Get a job by its ID or full URL.
|
|
87
88
|
|
|
88
89
|
Args:
|
|
89
|
-
job_id (str): The UUID of the job to retrieve
|
|
90
|
+
job_id (str): The UUID of the job to retrieve, or a full URL to the job.
|
|
91
|
+
Can be either:
|
|
92
|
+
- Job UUID (e.g., 'abc123')
|
|
93
|
+
- Full URL (e.g., 'https://biolib.com/result/abc123/?token=xyz789')
|
|
94
|
+
- Full URL with token parameter (e.g., 'biolib.com/result/abc123/token=xyz789')
|
|
90
95
|
job_token (str, optional): Authentication token for accessing the job.
|
|
91
96
|
Only needed for jobs that aren't owned by the current user.
|
|
97
|
+
If the URL contains a token, this parameter is ignored.
|
|
92
98
|
|
|
93
99
|
Returns:
|
|
94
100
|
Job: The job object
|
|
95
101
|
|
|
96
102
|
Example::
|
|
97
103
|
|
|
104
|
+
>>> # Get by UUID
|
|
98
105
|
>>> job = biolib.get_job('abc123')
|
|
99
|
-
>>> #
|
|
106
|
+
>>> # Get with explicit token
|
|
100
107
|
>>> job = biolib.get_job('abc123', job_token='xyz789')
|
|
108
|
+
>>> # Get by full URL with token
|
|
109
|
+
>>> job = biolib.get_job('https://biolib.com/result/abc123/?token=xyz789')
|
|
110
|
+
>>> # Get by URL with inline token format
|
|
111
|
+
>>> job = biolib.get_job('biolib.com/result/abc123/token=xyz789')
|
|
101
112
|
"""
|
|
102
|
-
|
|
113
|
+
uuid, token = _parse_result_id_or_url(job_id, job_token)
|
|
114
|
+
return _Result.create_from_uuid(uuid=uuid, auth_token=token)
|
|
103
115
|
|
|
104
116
|
|
|
105
117
|
def get_result(result_id: str, result_token: Optional[str] = None) -> _Result:
|
|
106
|
-
r"""Get a result by its ID.
|
|
118
|
+
r"""Get a result by its ID or full URL.
|
|
107
119
|
|
|
108
120
|
Args:
|
|
109
|
-
result_id (str): The UUID of the result to retrieve
|
|
121
|
+
result_id (str): The UUID of the result to retrieve, or a full URL to the result.
|
|
122
|
+
Can be either:
|
|
123
|
+
- Result UUID (e.g., 'abc123')
|
|
124
|
+
- Full URL (e.g., 'https://biolib.com/result/abc123/?token=xyz789')
|
|
125
|
+
- Full URL with token parameter (e.g., 'biolib.com/result/abc123/token=xyz789')
|
|
110
126
|
result_token (str, optional): Authentication token for accessing the result.
|
|
111
|
-
Only needed for
|
|
127
|
+
Only needed for results that aren't owned by the current user.
|
|
128
|
+
If the URL contains a token, this parameter is ignored.
|
|
112
129
|
|
|
113
130
|
Returns:
|
|
114
131
|
Result: The result object
|
|
115
132
|
|
|
116
133
|
Example::
|
|
117
134
|
|
|
135
|
+
>>> # Get by UUID
|
|
118
136
|
>>> result = biolib.get_result('abc123')
|
|
119
|
-
>>> #
|
|
137
|
+
>>> # Get with explicit token
|
|
120
138
|
>>> result = biolib.get_result('abc123', result_token='xyz789')
|
|
139
|
+
>>> # Get by full URL with token
|
|
140
|
+
>>> result = biolib.get_result('https://biolib.com/result/abc123/?token=xyz789')
|
|
141
|
+
>>> # Get by URL with inline token format
|
|
142
|
+
>>> result = biolib.get_result('biolib.com/result/abc123/token=xyz789')
|
|
121
143
|
"""
|
|
122
|
-
|
|
144
|
+
uuid, token = _parse_result_id_or_url(result_id, result_token)
|
|
145
|
+
return _Result.create_from_uuid(uuid=uuid, auth_token=token)
|
|
123
146
|
|
|
124
147
|
|
|
125
148
|
def get_data_record(uri: str) -> _DataRecord:
|
|
@@ -6,7 +6,6 @@ from struct import Struct
|
|
|
6
6
|
from typing import Callable, Dict, Iterable, List, Optional, Union, cast
|
|
7
7
|
|
|
8
8
|
from biolib import api
|
|
9
|
-
from biolib._internal import types
|
|
10
9
|
from biolib._internal.data_record import get_data_record_state_from_uri
|
|
11
10
|
from biolib._internal.data_record.data_record import validate_sqlite_v1
|
|
12
11
|
from biolib._internal.data_record.push_data import (
|
|
@@ -15,14 +14,15 @@ from biolib._internal.data_record.push_data import (
|
|
|
15
14
|
)
|
|
16
15
|
from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
|
|
17
16
|
from biolib._internal.http_client import HttpClient
|
|
18
|
-
from biolib.
|
|
17
|
+
from biolib._shared import types
|
|
18
|
+
from biolib._shared.types import ZipFileNodeDict
|
|
19
|
+
from biolib._shared.utils import parse_resource_uri
|
|
19
20
|
from biolib.api import client as api_client
|
|
20
21
|
from biolib.biolib_api_client import BiolibApiClient
|
|
21
22
|
from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersion, DataRecordVersionInfo
|
|
22
23
|
from biolib.biolib_binary_format import LazyLoadedFile
|
|
23
24
|
from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
|
|
24
25
|
from biolib.biolib_logging import logger
|
|
25
|
-
from biolib.utils.app_uri import parse_app_uri
|
|
26
26
|
|
|
27
27
|
PathFilter = Union[str, List[str], Callable[[str], bool]]
|
|
28
28
|
|
|
@@ -44,11 +44,11 @@ class DataRecord:
|
|
|
44
44
|
|
|
45
45
|
@property
|
|
46
46
|
def name(self) -> str:
|
|
47
|
-
uri_parsed =
|
|
48
|
-
if not uri_parsed['
|
|
47
|
+
uri_parsed = parse_resource_uri(self._state['resource_uri'], use_account_as_name_default=False)
|
|
48
|
+
if not uri_parsed['resource_name']:
|
|
49
49
|
raise ValueError('Expected parameter "resource_uri" to contain resource name')
|
|
50
50
|
|
|
51
|
-
return uri_parsed['
|
|
51
|
+
return uri_parsed['resource_name']
|
|
52
52
|
|
|
53
53
|
def list_files(
|
|
54
54
|
self,
|
|
@@ -142,8 +142,8 @@ class DataRecord:
|
|
|
142
142
|
BiolibApiClient.assert_is_signed_in(authenticated_action_description='create a Data Record')
|
|
143
143
|
if data_path is not None:
|
|
144
144
|
assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
|
|
145
|
-
uri_parsed =
|
|
146
|
-
if uri_parsed['
|
|
145
|
+
uri_parsed = parse_resource_uri(destination, use_account_as_name_default=False)
|
|
146
|
+
if uri_parsed['resource_name_normalized']:
|
|
147
147
|
data_record_uri = destination
|
|
148
148
|
else:
|
|
149
149
|
record_name = 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
|
|
@@ -173,10 +173,10 @@ class DataRecord:
|
|
|
173
173
|
'resource_type': 'data-record',
|
|
174
174
|
}
|
|
175
175
|
if uri:
|
|
176
|
-
uri_parsed =
|
|
176
|
+
uri_parsed = parse_resource_uri(uri, use_account_as_name_default=False)
|
|
177
177
|
params['account_handle'] = uri_parsed['account_handle_normalized']
|
|
178
|
-
if uri_parsed['
|
|
179
|
-
params['app_name'] = uri_parsed['
|
|
178
|
+
if uri_parsed['resource_name_normalized']:
|
|
179
|
+
params['app_name'] = uri_parsed['resource_name_normalized']
|
|
180
180
|
|
|
181
181
|
results = api_client.get(path='/apps/', params=params).json()['results']
|
|
182
182
|
if count is None and len(results) == max_page_size:
|
|
@@ -284,3 +284,16 @@ class DataRecord:
|
|
|
284
284
|
|
|
285
285
|
def _get_detailed_dict(self) -> types.DataRecordDetailedDict:
|
|
286
286
|
return cast(types.DataRecordDetailedDict, api_client.get(f'/resources/data-records/{self.uuid}/').json())
|
|
287
|
+
|
|
288
|
+
def delete(self) -> None:
|
|
289
|
+
"""Delete the data record.
|
|
290
|
+
|
|
291
|
+
Example::
|
|
292
|
+
>>> record = DataRecord.get_by_uri("account/data-record")
|
|
293
|
+
>>> record.delete()
|
|
294
|
+
"""
|
|
295
|
+
try:
|
|
296
|
+
api_client.delete(path=f'/apps/{self.uuid}/')
|
|
297
|
+
logger.info(f'Data record {self.uri} deleted')
|
|
298
|
+
except Exception as error:
|
|
299
|
+
raise Exception(f'Failed to delete data record {self.uri} due to: {error}') from error
|
|
File without changes
|
biolib/_index/index.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any, Dict
|
|
3
|
+
|
|
4
|
+
from biolib import api
|
|
5
|
+
from biolib._index.types import IndexInfo
|
|
6
|
+
from biolib._internal.index import get_index_from_uri
|
|
7
|
+
from biolib.biolib_api_client import BiolibApiClient
|
|
8
|
+
from biolib.biolib_logging import logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Index:
|
|
12
|
+
def __init__(self, _internal_state: IndexInfo):
|
|
13
|
+
self._state = _internal_state
|
|
14
|
+
|
|
15
|
+
def __repr__(self) -> str:
|
|
16
|
+
return f'Index: {self._state["resource_uri"]}'
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def uri(self) -> str:
|
|
20
|
+
return self._state['resource_uri']
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def id(self) -> str:
|
|
24
|
+
return f"{self._state['group_uuid']}.{self._state['resource_uuid']}".replace("-", "_")
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
def get_by_uri(uri: str) -> 'Index':
|
|
28
|
+
return Index(_internal_state=get_index_from_uri(uri))
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def create(uri: str, config: Dict[str, Any]) -> str:
|
|
32
|
+
BiolibApiClient.assert_is_signed_in(authenticated_action_description='create an Index')
|
|
33
|
+
|
|
34
|
+
response = api.client.post(
|
|
35
|
+
path='/resources/indexes/',
|
|
36
|
+
data={
|
|
37
|
+
'uri': uri,
|
|
38
|
+
'index_config': config,
|
|
39
|
+
},
|
|
40
|
+
)
|
|
41
|
+
result = response.json()
|
|
42
|
+
created_uri: str = result['uri']
|
|
43
|
+
logger.info(f"Successfully created Index '{created_uri}'")
|
|
44
|
+
return created_uri
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def create_from_config_file(uri: str, config_path: str) -> str:
|
|
48
|
+
with open(config_path) as config_file:
|
|
49
|
+
index_config = json.load(config_file)
|
|
50
|
+
|
|
51
|
+
return Index.create(uri=uri, config=index_config)
|
biolib/_index/types.py
ADDED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import sqlite3
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
|
-
from biolib.
|
|
4
|
+
from biolib._shared.types import SqliteV1DatabaseSchema
|
|
5
5
|
from biolib.api import client as api_client
|
|
6
6
|
from biolib.biolib_api_client import AppGetResponse
|
|
7
7
|
from biolib.biolib_api_client.biolib_app_api import _get_app_uri_from_str
|
|
@@ -2,9 +2,9 @@ import os
|
|
|
2
2
|
|
|
3
3
|
from biolib import utils
|
|
4
4
|
from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
|
|
5
|
-
from biolib._internal.types.typing import List, Optional, Tuple
|
|
6
5
|
from biolib.biolib_errors import BioLibError
|
|
7
6
|
from biolib.biolib_logging import logger
|
|
7
|
+
from biolib.typing_utils import List, Optional, Tuple
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def validate_data_path_and_get_files_and_size_of_directory(data_path: str) -> Tuple[List[str], int]:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
-
from datetime import datetime, timedelta
|
|
2
|
+
from datetime import datetime, timedelta, timezone
|
|
3
3
|
from urllib.parse import urlparse
|
|
4
4
|
|
|
5
5
|
from biolib.api import client as api_client
|
|
@@ -16,7 +16,7 @@ class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
|
|
|
16
16
|
self._presigned_url: Optional[str] = None
|
|
17
17
|
|
|
18
18
|
def get_remote_url(self) -> str:
|
|
19
|
-
if not self._presigned_url or not self._expires_at or datetime.
|
|
19
|
+
if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
|
|
20
20
|
lfs_version: DataRecordVersion = api_client.get(
|
|
21
21
|
path=f'/lfs/versions/{self._resource_version_uuid}/',
|
|
22
22
|
).json()
|
|
@@ -29,7 +29,7 @@ class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
|
|
|
29
29
|
else:
|
|
30
30
|
self._presigned_url = lfs_version['presigned_download_url']
|
|
31
31
|
|
|
32
|
-
self._expires_at = datetime.
|
|
32
|
+
self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
|
|
33
33
|
logger.debug(
|
|
34
34
|
f'DataRecord "{self._resource_version_uuid}" fetched presigned URL '
|
|
35
35
|
f'with expiry at {self._expires_at.isoformat()}'
|
biolib/_internal/file_utils.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import io
|
|
3
3
|
import os
|
|
4
|
+
import posixpath
|
|
4
5
|
import zipfile as zf
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
@@ -114,9 +115,11 @@ def path_to_renamed_path(path_str: str, prefix_with_slash: bool = True) -> str:
|
|
|
114
115
|
|
|
115
116
|
if prefix_with_slash:
|
|
116
117
|
if not result.startswith('/'):
|
|
117
|
-
|
|
118
|
-
|
|
118
|
+
result = '/' + result
|
|
119
|
+
# Normalize to handle cases like '/./mydir' -> '/mydir' and remove trailing slashes.
|
|
120
|
+
# Required because downstream Mappings class does exact string-prefix matching.
|
|
121
|
+
return posixpath.normpath(result)
|
|
119
122
|
else:
|
|
120
123
|
if result.startswith('/'):
|
|
121
|
-
|
|
122
|
-
return result
|
|
124
|
+
result = result[1:]
|
|
125
|
+
return posixpath.normpath(result)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .index import get_index_from_uri
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from typing import Any, Dict
|
|
2
|
+
|
|
3
|
+
from biolib._index.types import IndexInfo
|
|
4
|
+
from biolib.api import client as api_client
|
|
5
|
+
from biolib.biolib_api_client.biolib_app_api import _get_app_uri_from_str
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_index_from_uri(uri: str) -> IndexInfo:
|
|
9
|
+
normalized_uri = _get_app_uri_from_str(uri)
|
|
10
|
+
app_response: Dict[str, Any] = api_client.get(path='/app/', params={'uri': normalized_uri}).json()
|
|
11
|
+
resource_uri = app_response['app_version']['app_uri']
|
|
12
|
+
if app_response['app']['type'] != 'index':
|
|
13
|
+
raise Exception(f'Resource "{resource_uri}" is not an Index')
|
|
14
|
+
return IndexInfo(
|
|
15
|
+
resource_uri=app_response['app_version']['app_uri'],
|
|
16
|
+
resource_uuid=app_response['app']['public_id'],
|
|
17
|
+
group_uuid=app_response['app']['group_uuid'],
|
|
18
|
+
)
|
biolib/_internal/lfs/cache.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import subprocess
|
|
3
|
-
from datetime import datetime, timedelta
|
|
3
|
+
from datetime import datetime, timedelta, timezone
|
|
4
4
|
|
|
5
5
|
from biolib.biolib_logging import logger_no_user_data
|
|
6
6
|
from biolib.compute_node.job_worker.cache_state import LfsCacheState
|
|
@@ -9,7 +9,7 @@ from biolib.compute_node.job_worker.cache_state import LfsCacheState
|
|
|
9
9
|
def prune_lfs_cache(dry_run: bool) -> None:
|
|
10
10
|
logger_no_user_data.info(f'Pruning LFS cache (dry run = {dry_run})...')
|
|
11
11
|
|
|
12
|
-
current_time = datetime.
|
|
12
|
+
current_time = datetime.now(timezone.utc)
|
|
13
13
|
paths_to_delete = set()
|
|
14
14
|
|
|
15
15
|
with LfsCacheState() as state:
|
|
@@ -24,6 +24,8 @@ def prune_lfs_cache(dry_run: bool) -> None:
|
|
|
24
24
|
lfs_uuids_to_keep_in_state = set()
|
|
25
25
|
for lfs_uuid, lfs in state['large_file_systems'].items():
|
|
26
26
|
last_used_at = datetime.fromisoformat(lfs['last_used_at'])
|
|
27
|
+
if last_used_at.tzinfo is None:
|
|
28
|
+
last_used_at = last_used_at.replace(tzinfo=timezone.utc)
|
|
27
29
|
lfs_time_to_live_in_days = 60 if lfs['state'] == 'ready' else 7
|
|
28
30
|
|
|
29
31
|
if last_used_at < current_time - timedelta(days=lfs_time_to_live_in_days):
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
2
3
|
import re
|
|
3
4
|
import sys
|
|
@@ -14,14 +15,14 @@ from biolib._internal.data_record.push_data import (
|
|
|
14
15
|
)
|
|
15
16
|
from biolib._internal.errors import AuthenticationError
|
|
16
17
|
from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
|
|
17
|
-
from biolib.
|
|
18
|
+
from biolib._shared.types import PushResponseDict
|
|
19
|
+
from biolib._shared.utils import parse_resource_uri
|
|
18
20
|
from biolib.biolib_api_client import BiolibApiClient
|
|
19
21
|
from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
|
|
20
22
|
from biolib.biolib_docker_client import BiolibDockerClient
|
|
21
23
|
from biolib.biolib_errors import BioLibError
|
|
22
24
|
from biolib.biolib_logging import logger
|
|
23
|
-
from biolib.typing_utils import Iterable, Optional, Set, TypedDict
|
|
24
|
-
from biolib.utils.app_uri import parse_app_uri
|
|
25
|
+
from biolib.typing_utils import Dict, Iterable, Optional, Set, TypedDict, Union
|
|
25
26
|
|
|
26
27
|
REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')
|
|
27
28
|
|
|
@@ -108,8 +109,10 @@ def _process_docker_status_updates_with_progress_bar(status_updates: Iterable[Do
|
|
|
108
109
|
|
|
109
110
|
|
|
110
111
|
def _process_docker_status_updates_with_logging(status_updates: Iterable[DockerStatusUpdate], action: str) -> None:
|
|
111
|
-
layer_progress = {}
|
|
112
|
-
layer_status = {}
|
|
112
|
+
layer_progress: Dict[str, float] = {}
|
|
113
|
+
layer_status: Dict[str, str] = {}
|
|
114
|
+
layer_details: Dict[str, Dict[str, int]] = {}
|
|
115
|
+
layer_bytes_at_last_log: Dict[str, int] = {}
|
|
113
116
|
last_log_time = time.time()
|
|
114
117
|
|
|
115
118
|
logger.info(f'{action} Docker image...')
|
|
@@ -127,6 +130,7 @@ def _process_docker_status_updates_with_logging(status_updates: Iterable[DockerS
|
|
|
127
130
|
percentage = (current / total * 100) if total > 0 else 0
|
|
128
131
|
layer_progress[layer_id] = percentage
|
|
129
132
|
layer_status[layer_id] = f'{action.lower()}'
|
|
133
|
+
layer_details[layer_id] = {'current': current, 'total': total}
|
|
130
134
|
elif update.get('status') == 'Layer already exists':
|
|
131
135
|
layer_progress[layer_id] = 100
|
|
132
136
|
layer_status[layer_id] = 'already exists'
|
|
@@ -145,16 +149,33 @@ def _process_docker_status_updates_with_logging(status_updates: Iterable[DockerS
|
|
|
145
149
|
logger.info(f'{action} Docker image - {status}')
|
|
146
150
|
|
|
147
151
|
if current_time - last_log_time >= 10.0:
|
|
148
|
-
_log_progress_summary(
|
|
152
|
+
_log_progress_summary(
|
|
153
|
+
action,
|
|
154
|
+
layer_progress,
|
|
155
|
+
layer_status,
|
|
156
|
+
layer_details,
|
|
157
|
+
layer_bytes_at_last_log,
|
|
158
|
+
current_time - last_log_time,
|
|
159
|
+
)
|
|
160
|
+
layer_bytes_at_last_log = {lid: details['current'] for lid, details in layer_details.items()}
|
|
149
161
|
last_log_time = current_time
|
|
150
162
|
|
|
151
|
-
_log_progress_summary(
|
|
163
|
+
_log_progress_summary(
|
|
164
|
+
action, layer_progress, layer_status, layer_details, layer_bytes_at_last_log, time.time() - last_log_time
|
|
165
|
+
)
|
|
152
166
|
if action == 'Pushing':
|
|
153
167
|
logger.info('Pushing final image manifest...')
|
|
154
168
|
logger.info(f'{action} Docker image completed')
|
|
155
169
|
|
|
156
170
|
|
|
157
|
-
def _log_progress_summary(
|
|
171
|
+
def _log_progress_summary(
|
|
172
|
+
action: str,
|
|
173
|
+
layer_progress: Dict[str, float],
|
|
174
|
+
layer_status: Dict[str, str],
|
|
175
|
+
layer_details: Dict[str, Dict[str, int]],
|
|
176
|
+
layer_bytes_at_last_log: Dict[str, int],
|
|
177
|
+
time_delta: float,
|
|
178
|
+
) -> None:
|
|
158
179
|
if not layer_progress and not layer_status:
|
|
159
180
|
return
|
|
160
181
|
|
|
@@ -173,7 +194,36 @@ def _log_progress_summary(action: str, layer_progress: dict, layer_status: dict)
|
|
|
173
194
|
if status in ['preparing', 'waiting', 'pushing', 'uploading'] and layer_progress.get(layer_id, 0) < 100
|
|
174
195
|
]
|
|
175
196
|
|
|
176
|
-
if active_layers:
|
|
197
|
+
if active_layers and layer_details:
|
|
198
|
+
total_bytes_transferred = 0
|
|
199
|
+
layer_info_parts = []
|
|
200
|
+
|
|
201
|
+
for layer_id in active_layers[:5]:
|
|
202
|
+
if layer_id in layer_details:
|
|
203
|
+
details = layer_details[layer_id]
|
|
204
|
+
current = details['current']
|
|
205
|
+
total = details['total']
|
|
206
|
+
percentage = layer_progress.get(layer_id, 0)
|
|
207
|
+
|
|
208
|
+
bytes_since_last = current - layer_bytes_at_last_log.get(layer_id, 0)
|
|
209
|
+
total_bytes_transferred += bytes_since_last
|
|
210
|
+
|
|
211
|
+
current_mb = current / (1024 * 1024)
|
|
212
|
+
total_mb = total / (1024 * 1024)
|
|
213
|
+
layer_info_parts.append(f'{layer_id}: {current_mb:.1f}/{total_mb:.1f} MB ({percentage:.1f}%)')
|
|
214
|
+
|
|
215
|
+
speed_info = ''
|
|
216
|
+
if time_delta > 0 and total_bytes_transferred > 0:
|
|
217
|
+
speed_mbps = (total_bytes_transferred / (1024 * 1024)) / time_delta
|
|
218
|
+
speed_info = f' @ {speed_mbps:.2f} MB/s'
|
|
219
|
+
|
|
220
|
+
more_layers_info = ''
|
|
221
|
+
if len(active_layers) > 5:
|
|
222
|
+
more_layers_info = f' (+ {len(active_layers) - 5} more)'
|
|
223
|
+
|
|
224
|
+
if layer_info_parts:
|
|
225
|
+
logger.info(f'Active layers: {", ".join(layer_info_parts)}{speed_info}{more_layers_info}')
|
|
226
|
+
elif active_layers:
|
|
177
227
|
logger.info(f'Active layers: {", ".join(active_layers[:5])}{"..." if len(active_layers) > 5 else ""}')
|
|
178
228
|
|
|
179
229
|
|
|
@@ -195,13 +245,12 @@ def push_application(
|
|
|
195
245
|
set_as_published: bool,
|
|
196
246
|
dry_run: bool = False,
|
|
197
247
|
) -> Optional[PushResponseDict]:
|
|
198
|
-
|
|
199
|
-
|
|
248
|
+
app_uri = app_uri.rstrip('/')
|
|
249
|
+
parsed_uri = parse_resource_uri(app_uri)
|
|
250
|
+
resource_name = parsed_uri['resource_name']
|
|
200
251
|
|
|
201
|
-
app_uri_prefix =
|
|
202
|
-
|
|
203
|
-
)
|
|
204
|
-
app_uri_to_fetch = f"{app_uri_prefix}{parsed_uri['account_handle_normalized']}/{app_name}"
|
|
252
|
+
app_uri_prefix = f"@{parsed_uri['resource_prefix']}/" if parsed_uri['resource_prefix'] is not None else ''
|
|
253
|
+
app_uri_to_fetch = f"{app_uri_prefix}{parsed_uri['account_handle_normalized']}/{resource_name}"
|
|
205
254
|
|
|
206
255
|
version = parsed_uri['version']
|
|
207
256
|
semantic_version = f"{version['major']}.{version['minor']}.{version['patch']}" if version else None
|
|
@@ -238,19 +287,34 @@ def push_application(
|
|
|
238
287
|
app_data_path: Optional[Path] = None
|
|
239
288
|
try:
|
|
240
289
|
with open(config_yml_path) as config_yml_file:
|
|
241
|
-
|
|
290
|
+
try:
|
|
291
|
+
config = json.loads(json.dumps(yaml.safe_load(config_yml_file.read())))
|
|
292
|
+
except (TypeError, ValueError) as e:
|
|
293
|
+
raise BioLibError(
|
|
294
|
+
f'The .biolib/config.yml file contains data types that are not supported '
|
|
295
|
+
f'(must be JSON-serializable). Please ensure only standard JSON types '
|
|
296
|
+
f'(str, int, float, bool, list, dict, null) are used. Original error: {e}'
|
|
297
|
+
) from e
|
|
298
|
+
|
|
299
|
+
if 'assets' in config and 'app_data' not in config:
|
|
300
|
+
config['app_data'] = config.pop('assets')
|
|
301
|
+
elif 'assets' in config and 'app_data' in config:
|
|
302
|
+
raise BioLibError(
|
|
303
|
+
'In .biolib/config.yml you cannot specify both "app_data" and "assets" fields. Please use only one.'
|
|
304
|
+
)
|
|
242
305
|
|
|
243
306
|
app_data = config.get('app_data')
|
|
244
307
|
if app_data:
|
|
308
|
+
field_name = 'app_data' if 'app_data' in config else 'assets'
|
|
245
309
|
if not isinstance(app_data, str):
|
|
246
310
|
raise BioLibError(
|
|
247
|
-
f'In .biolib/config.yml the value of "
|
|
311
|
+
f'In .biolib/config.yml the value of "{field_name}" must be a string but got {type(app_data)}'
|
|
248
312
|
)
|
|
249
313
|
|
|
250
314
|
app_data_path = app_path_absolute.joinpath(app_data).resolve()
|
|
251
315
|
if not app_data_path.is_dir():
|
|
252
316
|
raise BioLibError(
|
|
253
|
-
'In .biolib/config.yml the value of "
|
|
317
|
+
f'In .biolib/config.yml the value of "{field_name}" must be a path to a directory '
|
|
254
318
|
'in the application directory'
|
|
255
319
|
)
|
|
256
320
|
|
|
@@ -321,10 +385,6 @@ def push_application(
|
|
|
321
385
|
app_response = BiolibAppApi.get_by_uri(app_uri_to_fetch)
|
|
322
386
|
app = app_response['app']
|
|
323
387
|
|
|
324
|
-
if app_data and not app['allow_client_side_execution']:
|
|
325
|
-
raise BioLibError(
|
|
326
|
-
'To push a version with app_data the app must be set to "Allow Client-Side Source Code Access"'
|
|
327
|
-
)
|
|
328
388
|
if dry_run:
|
|
329
389
|
logger.info('Successfully completed dry-run. No new version was pushed.')
|
|
330
390
|
return None
|
|
@@ -410,9 +470,15 @@ def push_application(
|
|
|
410
470
|
logger.info(f'Successfully pushed {docker_image_name}')
|
|
411
471
|
|
|
412
472
|
app_version_uuid = new_app_version_json['public_id']
|
|
473
|
+
complete_push_data: Dict[str, Union[bool, str]] = {
|
|
474
|
+
'set_as_active': set_as_active,
|
|
475
|
+
'set_as_published': set_as_published,
|
|
476
|
+
}
|
|
477
|
+
if parsed_uri['tag']:
|
|
478
|
+
complete_push_data['tag'] = parsed_uri['tag']
|
|
413
479
|
api.client.post(
|
|
414
480
|
path=f'/app-versions/{app_version_uuid}/complete-push/',
|
|
415
|
-
data=
|
|
481
|
+
data=complete_push_data,
|
|
416
482
|
)
|
|
417
483
|
|
|
418
484
|
sematic_version = f"{new_app_version_json['major']}.{new_app_version_json['minor']}.{new_app_version_json['patch']}"
|
biolib/_internal/runtime.py
CHANGED
|
@@ -4,10 +4,12 @@ from biolib.typing_utils import TypedDict
|
|
|
4
4
|
class RuntimeJobDataDict(TypedDict):
|
|
5
5
|
version: str
|
|
6
6
|
job_requested_machine: str
|
|
7
|
+
job_requested_machine_spot: bool
|
|
7
8
|
job_uuid: str
|
|
8
9
|
job_auth_token: str
|
|
9
10
|
app_uri: str
|
|
10
11
|
is_environment_biolib_cloud: bool
|
|
12
|
+
job_reserved_machines: int
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class BioLibRuntimeError(Exception):
|
|
@@ -1,16 +1,52 @@
|
|
|
1
|
+
import { useState, useEffect } from "react";
|
|
2
|
+
import biolib from "./biolib-sdk";
|
|
3
|
+
|
|
1
4
|
export default function App() {
|
|
5
|
+
const [outputFileData, setOutputFileData] = useState<Uint8Array | null>(null);
|
|
6
|
+
const [loading, setLoading] = useState(true);
|
|
7
|
+
|
|
8
|
+
const loadOutputData = async () => {
|
|
9
|
+
setLoading(true);
|
|
10
|
+
try {
|
|
11
|
+
const data = await biolib.getOutputFileData("output.json");
|
|
12
|
+
setOutputFileData(data);
|
|
13
|
+
} catch (error) {
|
|
14
|
+
console.error("Error loading output data:", error);
|
|
15
|
+
setOutputFileData(null);
|
|
16
|
+
} finally {
|
|
17
|
+
setLoading(false);
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
useEffect(() => {
|
|
22
|
+
loadOutputData();
|
|
23
|
+
}, []);
|
|
24
|
+
|
|
2
25
|
return (
|
|
3
26
|
<div className="min-h-screen bg-gray-100 flex items-center justify-center">
|
|
4
|
-
<div className="text-center">
|
|
27
|
+
<div className="text-center max-w-2xl mx-auto p-8">
|
|
5
28
|
<h1 className="text-4xl font-bold mb-4">
|
|
6
29
|
Hello, BioLib!
|
|
7
30
|
</h1>
|
|
8
31
|
<p className="text-lg mb-2">
|
|
9
32
|
You have successfully set up your BioLib GUI application.
|
|
10
33
|
</p>
|
|
11
|
-
<p className="italic">
|
|
34
|
+
<p className="italic mb-6">
|
|
12
35
|
This is a simple React template with Tailwind CSS styling.
|
|
13
36
|
</p>
|
|
37
|
+
|
|
38
|
+
<div className="mt-8 p-4 bg-white rounded-lg shadow">
|
|
39
|
+
<h2 className="text-xl font-semibold mb-4">Example: Reading Output Files</h2>
|
|
40
|
+
{loading ? (
|
|
41
|
+
<p className="text-gray-500">Loading output.json...</p>
|
|
42
|
+
) : outputFileData ? (
|
|
43
|
+
<div className="p-3 bg-gray-50 rounded text-left">
|
|
44
|
+
<pre className="text-sm">{new TextDecoder().decode(outputFileData)}</pre>
|
|
45
|
+
</div>
|
|
46
|
+
) : (
|
|
47
|
+
<p className="text-red-500">Failed to load output.json</p>
|
|
48
|
+
)}
|
|
49
|
+
</div>
|
|
14
50
|
</div>
|
|
15
51
|
</div>
|
|
16
52
|
);
|