pybiolib 1.2.1056__py3-none-any.whl → 1.2.1727__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pybiolib might be problematic. Click here for more details.
- biolib/__init__.py +33 -10
- biolib/_data_record/data_record.py +103 -26
- biolib/_index/__init__.py +0 -0
- biolib/_index/index.py +51 -0
- biolib/_index/types.py +7 -0
- biolib/_internal/data_record/data_record.py +1 -1
- biolib/_internal/data_record/push_data.py +65 -16
- biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
- biolib/_internal/file_utils.py +7 -4
- biolib/_internal/index/__init__.py +1 -0
- biolib/_internal/index/index.py +18 -0
- biolib/_internal/lfs/cache.py +4 -2
- biolib/_internal/push_application.py +89 -23
- biolib/_internal/runtime.py +2 -0
- biolib/_internal/templates/gui_template/App.tsx +38 -2
- biolib/_internal/templates/gui_template/Dockerfile +2 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/package.json +1 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
- biolib/_internal/templates/gui_template/vite.config.mts +2 -1
- biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
- biolib/_internal/templates/init_template/Dockerfile +2 -0
- biolib/_internal/utils/__init__.py +40 -0
- biolib/_internal/utils/auth.py +46 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_runtime/runtime.py +9 -0
- biolib/_session/session.py +7 -5
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +74 -0
- biolib/_shared/types/resource.py +37 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/{_internal → _shared}/types/resource_version.py +8 -2
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/client.py +3 -47
- biolib/app/app.py +57 -33
- biolib/biolib_api_client/api_client.py +3 -47
- biolib/biolib_api_client/app_types.py +1 -6
- biolib/biolib_api_client/biolib_app_api.py +17 -0
- biolib/biolib_binary_format/module_input.py +8 -0
- biolib/biolib_binary_format/remote_endpoints.py +3 -3
- biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
- biolib/cli/__init__.py +2 -1
- biolib/cli/data_record.py +82 -0
- biolib/cli/index.py +32 -0
- biolib/cli/init.py +39 -1
- biolib/cli/lfs.py +1 -1
- biolib/cli/run.py +8 -5
- biolib/cli/start.py +14 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
- biolib/compute_node/job_worker/executors/docker_types.py +1 -1
- biolib/compute_node/job_worker/executors/types.py +6 -5
- biolib/compute_node/job_worker/job_worker.py +149 -93
- biolib/compute_node/job_worker/large_file_system.py +2 -6
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
- biolib/compute_node/remote_host_proxy.py +139 -79
- biolib/compute_node/utils.py +2 -0
- biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +64 -19
- biolib/experiments/experiment.py +111 -16
- biolib/jobs/job.py +119 -29
- biolib/jobs/job_result.py +70 -33
- biolib/jobs/types.py +1 -0
- biolib/sdk/__init__.py +17 -2
- biolib/typing_utils.py +1 -1
- biolib/utils/cache_state.py +2 -2
- biolib/utils/multipart_uploader.py +24 -18
- biolib/utils/seq_util.py +1 -1
- pybiolib-1.2.1727.dist-info/METADATA +41 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1727.dist-info}/RECORD +103 -85
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1727.dist-info}/WHEEL +1 -1
- pybiolib-1.2.1727.dist-info/entry_points.txt +2 -0
- biolib/_internal/types/__init__.py +0 -6
- biolib/_internal/types/resource.py +0 -18
- biolib/utils/app_uri.py +0 -57
- pybiolib-1.2.1056.dist-info/METADATA +0 -50
- pybiolib-1.2.1056.dist-info/entry_points.txt +0 -3
- /biolib/{_internal → _shared}/types/account.py +0 -0
- /biolib/{_internal → _shared}/types/account_member.py +0 -0
- /biolib/{_internal → _shared}/types/app.py +0 -0
- /biolib/{_internal → _shared}/types/data_record.py +0 -0
- /biolib/{_internal → _shared}/types/experiment.py +0 -0
- /biolib/{_internal → _shared}/types/file_node.py +0 -0
- /biolib/{_internal → _shared}/types/push.py +0 -0
- /biolib/{_internal → _shared}/types/resource_permission.py +0 -0
- /biolib/{_internal → _shared}/types/result.py +0 -0
- /biolib/{_internal → _shared}/types/typing.py +0 -0
- /biolib/{_internal → _shared}/types/user.py +0 -0
- {pybiolib-1.2.1056.dist-info → pybiolib-1.2.1727.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from .account import AccountDict, AccountDetailedDict
|
|
2
|
+
from .account_member import AccountMemberDict
|
|
3
|
+
from .app import AppDetailedDict, AppSlimDict
|
|
4
|
+
from .data_record import (
|
|
5
|
+
DataRecordDetailedDict,
|
|
6
|
+
DataRecordSlimDict,
|
|
7
|
+
DataRecordTypeDict,
|
|
8
|
+
DataRecordValidationRuleDict,
|
|
9
|
+
SqliteV1Column,
|
|
10
|
+
SqliteV1DatabaseSchema,
|
|
11
|
+
SqliteV1ForeignKey,
|
|
12
|
+
SqliteV1Table,
|
|
13
|
+
)
|
|
14
|
+
from .experiment import (
|
|
15
|
+
DeprecatedExperimentDict,
|
|
16
|
+
ExperimentDetailedDict,
|
|
17
|
+
ExperimentDict,
|
|
18
|
+
ResultCounts,
|
|
19
|
+
)
|
|
20
|
+
from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
|
|
21
|
+
from .push import PushResponseDict
|
|
22
|
+
from .resource import ResourceDetailedDict, ResourceDict, ResourceTypeLiteral, ResourceUriDict, SemanticVersionDict
|
|
23
|
+
from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
|
|
24
|
+
from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
|
|
25
|
+
from .resource_version import (
|
|
26
|
+
ResourceVersionAssetsDict,
|
|
27
|
+
ResourceVersionDetailedDict,
|
|
28
|
+
ResourceVersionDict,
|
|
29
|
+
)
|
|
30
|
+
from .result import ResultDetailedDict, ResultDict
|
|
31
|
+
from .typing import Optional
|
|
32
|
+
from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
'AccountDetailedDict',
|
|
36
|
+
'AccountDict',
|
|
37
|
+
'AccountMemberDict',
|
|
38
|
+
'AppDetailedDict',
|
|
39
|
+
'AppSlimDict',
|
|
40
|
+
'DataRecordDetailedDict',
|
|
41
|
+
'DataRecordSlimDict',
|
|
42
|
+
'DataRecordTypeDict',
|
|
43
|
+
'DataRecordValidationRuleDict',
|
|
44
|
+
'DeprecatedExperimentDict',
|
|
45
|
+
'EnterpriseSettingsDict',
|
|
46
|
+
'ExperimentDetailedDict',
|
|
47
|
+
'ExperimentDict',
|
|
48
|
+
'FileNodeDict',
|
|
49
|
+
'FileZipMetadataDict',
|
|
50
|
+
'Optional',
|
|
51
|
+
'PushResponseDict',
|
|
52
|
+
'ResourceDeployKeyDict',
|
|
53
|
+
'ResourceDeployKeyWithSecretDict',
|
|
54
|
+
'ResourceDetailedDict',
|
|
55
|
+
'ResourceDict',
|
|
56
|
+
'ResourceTypeLiteral',
|
|
57
|
+
'ResourcePermissionDetailedDict',
|
|
58
|
+
'ResourcePermissionDict',
|
|
59
|
+
'ResourceUriDict',
|
|
60
|
+
'ResourceVersionAssetsDict',
|
|
61
|
+
'ResourceVersionDetailedDict',
|
|
62
|
+
'ResourceVersionDict',
|
|
63
|
+
'ResultCounts',
|
|
64
|
+
'ResultDetailedDict',
|
|
65
|
+
'ResultDict',
|
|
66
|
+
'SemanticVersionDict',
|
|
67
|
+
'SqliteV1Column',
|
|
68
|
+
'SqliteV1DatabaseSchema',
|
|
69
|
+
'SqliteV1ForeignKey',
|
|
70
|
+
'SqliteV1Table',
|
|
71
|
+
'UserDetailedDict',
|
|
72
|
+
'UserDict',
|
|
73
|
+
'ZipFileNodeDict',
|
|
74
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from .experiment import DeprecatedExperimentDict
|
|
2
|
+
from .resource_version import ResourceVersionDetailedDict
|
|
3
|
+
from .typing import Literal, NotRequired, Optional, TypedDict
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class SemanticVersionDict(TypedDict):
|
|
7
|
+
major: int
|
|
8
|
+
minor: int
|
|
9
|
+
patch: int
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ResourceUriDict(TypedDict):
|
|
13
|
+
account_handle_normalized: str
|
|
14
|
+
account_handle: str
|
|
15
|
+
resource_name_normalized: Optional[str]
|
|
16
|
+
resource_name: Optional[str]
|
|
17
|
+
resource_prefix: Optional[str]
|
|
18
|
+
version: Optional[SemanticVersionDict]
|
|
19
|
+
tag: Optional[str]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
ResourceTypeLiteral = Literal['app', 'data-record', 'experiment', 'index']
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ResourceDict(TypedDict):
|
|
26
|
+
uuid: str
|
|
27
|
+
uri: str
|
|
28
|
+
name: str
|
|
29
|
+
created_at: str
|
|
30
|
+
description: str
|
|
31
|
+
account_uuid: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ResourceDetailedDict(ResourceDict):
|
|
35
|
+
type: ResourceTypeLiteral
|
|
36
|
+
version: NotRequired[ResourceVersionDetailedDict]
|
|
37
|
+
experiment: Optional[DeprecatedExperimentDict]
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
from .typing import Literal, NotRequired, TypedDict
|
|
1
|
+
from .typing import Literal, NotRequired, Optional, TypedDict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ResourceVersionAssetsDict(TypedDict):
|
|
5
|
+
download_url: str
|
|
6
|
+
size_bytes: int
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
class ResourceVersionDict(TypedDict):
|
|
@@ -7,7 +12,8 @@ class ResourceVersionDict(TypedDict):
|
|
|
7
12
|
state: Literal['published', 'unpublished']
|
|
8
13
|
created_at: str
|
|
9
14
|
git_branch_name: NotRequired[str]
|
|
15
|
+
git_commit_hash: NotRequired[str]
|
|
10
16
|
|
|
11
17
|
|
|
12
18
|
class ResourceVersionDetailedDict(ResourceVersionDict):
|
|
13
|
-
|
|
19
|
+
assets: Optional[ResourceVersionAssetsDict]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from biolib._shared.types import Optional
|
|
4
|
+
from biolib._shared.types.resource import ResourceUriDict, SemanticVersionDict
|
|
5
|
+
from biolib.biolib_errors import BioLibError
|
|
6
|
+
|
|
7
|
+
URI_REGEX = re.compile(
|
|
8
|
+
r'^(@(?P<resource_prefix>[\w._-]+)/)?'
|
|
9
|
+
r'(?P<account_handle>[\w-]+)'
|
|
10
|
+
r'(/(?P<resource_name>[\w-]+))?'
|
|
11
|
+
r'(?::(?P<suffix>[^:]+))?$'
|
|
12
|
+
)
|
|
13
|
+
SEMVER_REGEX = re.compile(r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
|
|
14
|
+
TAG_REGEX = re.compile(r'^[a-z0-9-]{1,128}$')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def normalize_resource_name(string: str) -> str:
|
|
18
|
+
return string.replace('-', '_').lower()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_semantic_version(semantic_version: str) -> SemanticVersionDict:
|
|
22
|
+
if match := SEMVER_REGEX.fullmatch(semantic_version):
|
|
23
|
+
return SemanticVersionDict(
|
|
24
|
+
major=int(match.group('major')),
|
|
25
|
+
minor=int(match.group('minor')),
|
|
26
|
+
patch=int(match.group('patch')),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
raise ValueError('The version must be a valid semantic version in the format of major.minor.patch (1.2.3).')
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_resource_uri(uri: str, use_account_as_name_default: bool = True) -> ResourceUriDict:
|
|
33
|
+
matches = URI_REGEX.match(uri)
|
|
34
|
+
if matches is None:
|
|
35
|
+
raise BioLibError(f"Could not parse resource uri '{uri}', uri did not match regex")
|
|
36
|
+
|
|
37
|
+
version: Optional[SemanticVersionDict] = None
|
|
38
|
+
tag: Optional[str] = None
|
|
39
|
+
|
|
40
|
+
suffix = matches.group('suffix')
|
|
41
|
+
if suffix and suffix != '*':
|
|
42
|
+
try:
|
|
43
|
+
version = parse_semantic_version(suffix)
|
|
44
|
+
except ValueError:
|
|
45
|
+
if TAG_REGEX.fullmatch(suffix):
|
|
46
|
+
tag = suffix
|
|
47
|
+
else:
|
|
48
|
+
raise BioLibError(
|
|
49
|
+
f'Invalid version or tag "{suffix}". '
|
|
50
|
+
'Versions must be semantic versions like "1.2.3". '
|
|
51
|
+
'Tags must be lowercase alphanumeric or dashes and at most 128 characters.'
|
|
52
|
+
) from None
|
|
53
|
+
|
|
54
|
+
resource_prefix_raw: Optional[str] = matches.group('resource_prefix')
|
|
55
|
+
resource_prefix = resource_prefix_raw.lower() if resource_prefix_raw is not None else None
|
|
56
|
+
account_handle: str = matches.group('account_handle')
|
|
57
|
+
account_handle_normalized: str = normalize_resource_name(account_handle)
|
|
58
|
+
resource_name: Optional[str] = matches.group('resource_name')
|
|
59
|
+
|
|
60
|
+
if resource_name:
|
|
61
|
+
resource_name_normalized = normalize_resource_name(resource_name)
|
|
62
|
+
elif use_account_as_name_default:
|
|
63
|
+
resource_name_normalized = account_handle_normalized
|
|
64
|
+
else:
|
|
65
|
+
resource_name_normalized = None
|
|
66
|
+
|
|
67
|
+
return ResourceUriDict(
|
|
68
|
+
resource_prefix=resource_prefix,
|
|
69
|
+
account_handle=account_handle,
|
|
70
|
+
account_handle_normalized=account_handle_normalized,
|
|
71
|
+
resource_name_normalized=resource_name_normalized,
|
|
72
|
+
resource_name=resource_name if resource_name is not None or not use_account_as_name_default else account_handle,
|
|
73
|
+
version=version,
|
|
74
|
+
tag=tag,
|
|
75
|
+
)
|
biolib/api/client.py
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
import binascii
|
|
3
|
-
import json
|
|
4
1
|
from datetime import datetime, timezone
|
|
5
2
|
from json.decoder import JSONDecodeError
|
|
6
3
|
from urllib.parse import urlencode, urljoin
|
|
@@ -8,7 +5,8 @@ from urllib.parse import urlencode, urljoin
|
|
|
8
5
|
import importlib_metadata
|
|
9
6
|
|
|
10
7
|
from biolib._internal.http_client import HttpClient, HttpResponse
|
|
11
|
-
from biolib._internal.
|
|
8
|
+
from biolib._internal.utils.auth import decode_jwt_without_checking_signature
|
|
9
|
+
from biolib._shared.types.typing import Dict, Optional, TypedDict, Union, cast
|
|
12
10
|
from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
|
|
13
11
|
from biolib.biolib_errors import BioLibError
|
|
14
12
|
from biolib.biolib_logging import logger
|
|
@@ -33,10 +31,6 @@ class ApiClientInitDict(TypedDict):
|
|
|
33
31
|
client_type: Optional[str]
|
|
34
32
|
|
|
35
33
|
|
|
36
|
-
class JwtDecodeError(Exception):
|
|
37
|
-
pass
|
|
38
|
-
|
|
39
|
-
|
|
40
34
|
class ApiClient(HttpClient):
|
|
41
35
|
_biolib_package_version: str = _get_biolib_package_version()
|
|
42
36
|
|
|
@@ -147,7 +141,7 @@ class ApiClient(HttpClient):
|
|
|
147
141
|
|
|
148
142
|
def _get_access_token(self) -> str:
|
|
149
143
|
if self._access_token:
|
|
150
|
-
decoded_token =
|
|
144
|
+
decoded_token = decode_jwt_without_checking_signature(self._access_token)
|
|
151
145
|
if datetime.now(tz=timezone.utc).timestamp() < decoded_token['payload']['exp'] - 60: # 60 second buffer
|
|
152
146
|
# Token has not expired yet
|
|
153
147
|
return self._access_token
|
|
@@ -171,41 +165,3 @@ class ApiClient(HttpClient):
|
|
|
171
165
|
|
|
172
166
|
self._access_token = cast(str, response_dict['access'])
|
|
173
167
|
return self._access_token
|
|
174
|
-
|
|
175
|
-
@staticmethod
|
|
176
|
-
def _decode_jwt_without_checking_signature(jwt: str) -> Dict[str, Any]:
|
|
177
|
-
jwt_bytes = jwt.encode('utf-8')
|
|
178
|
-
|
|
179
|
-
try:
|
|
180
|
-
signing_input, _ = jwt_bytes.rsplit(b'.', 1)
|
|
181
|
-
header_segment, payload_segment = signing_input.split(b'.', 1)
|
|
182
|
-
except ValueError as error:
|
|
183
|
-
raise JwtDecodeError('Not enough segments') from error
|
|
184
|
-
|
|
185
|
-
try:
|
|
186
|
-
header_data = base64.urlsafe_b64decode(header_segment)
|
|
187
|
-
except (TypeError, binascii.Error) as error:
|
|
188
|
-
raise JwtDecodeError('Invalid header padding') from error
|
|
189
|
-
|
|
190
|
-
try:
|
|
191
|
-
header = json.loads(header_data)
|
|
192
|
-
except ValueError as error:
|
|
193
|
-
raise JwtDecodeError(f'Invalid header string: {error}') from error
|
|
194
|
-
|
|
195
|
-
if not isinstance(header, dict):
|
|
196
|
-
raise JwtDecodeError('Invalid header string: must be a json object')
|
|
197
|
-
|
|
198
|
-
try:
|
|
199
|
-
payload_data = base64.urlsafe_b64decode(payload_segment)
|
|
200
|
-
except (TypeError, binascii.Error) as error:
|
|
201
|
-
raise JwtDecodeError('Invalid payload padding') from error
|
|
202
|
-
|
|
203
|
-
try:
|
|
204
|
-
payload = json.loads(payload_data)
|
|
205
|
-
except ValueError as error:
|
|
206
|
-
raise JwtDecodeError(f'Invalid payload string: {error}') from error
|
|
207
|
-
|
|
208
|
-
if not isinstance(header, dict):
|
|
209
|
-
raise JwtDecodeError('Invalid payload string: must be a json object')
|
|
210
|
-
|
|
211
|
-
return dict(header=header, payload=payload)
|
biolib/app/app.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
import io
|
|
2
3
|
import json
|
|
3
4
|
import os
|
|
5
|
+
import posixpath
|
|
4
6
|
import random
|
|
5
7
|
import string
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
|
|
8
10
|
from biolib import utils
|
|
11
|
+
from biolib._internal.file_utils import path_to_renamed_path
|
|
12
|
+
from biolib._runtime.runtime import Runtime
|
|
13
|
+
from biolib._shared.utils import parse_resource_uri
|
|
9
14
|
from biolib.api.client import ApiClient
|
|
10
15
|
from biolib.biolib_api_client import JobState
|
|
11
16
|
from biolib.biolib_api_client.app_types import App, AppVersion
|
|
@@ -18,14 +23,24 @@ from biolib.compute_node.job_worker.job_worker import JobWorker
|
|
|
18
23
|
from biolib.experiments.experiment import Experiment
|
|
19
24
|
from biolib.jobs.job import Result
|
|
20
25
|
from biolib.typing_utils import Dict, Optional
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class JsonStringIO(io.StringIO):
|
|
29
|
+
pass
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
class BioLibApp:
|
|
27
|
-
def __init__(
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
uri: str,
|
|
36
|
+
_api_client: Optional[ApiClient] = None,
|
|
37
|
+
suppress_version_warning: bool = False,
|
|
38
|
+
_experiment: Optional[str] = None,
|
|
39
|
+
):
|
|
28
40
|
self._api_client: Optional[ApiClient] = _api_client
|
|
41
|
+
self._experiment = _experiment
|
|
42
|
+
self._input_uri = uri
|
|
43
|
+
self._parsed_input_uri = parse_resource_uri(uri)
|
|
29
44
|
|
|
30
45
|
app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
|
|
31
46
|
self._app: App = app_response['app']
|
|
@@ -33,16 +48,19 @@ class BioLibApp:
|
|
|
33
48
|
self._app_version: AppVersion = app_response['app_version']
|
|
34
49
|
|
|
35
50
|
if not suppress_version_warning:
|
|
36
|
-
|
|
37
|
-
if parsed_uri['version'] is None:
|
|
51
|
+
if self._parsed_input_uri['version'] is None:
|
|
38
52
|
if Runtime.check_is_environment_biolib_app():
|
|
39
53
|
logger.warning(
|
|
40
54
|
f"No version specified in URI '{uri}'. This will use the default version, "
|
|
41
|
-
f
|
|
55
|
+
f'which may change behaviour over time. Consider locking down the exact version, '
|
|
42
56
|
f"e.g. '{uri}:1.2.3'"
|
|
43
57
|
)
|
|
44
58
|
|
|
45
|
-
|
|
59
|
+
if self._parsed_input_uri['tag']:
|
|
60
|
+
semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
|
|
61
|
+
logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
|
|
62
|
+
else:
|
|
63
|
+
logger.info(f'Loaded {self._app_uri}')
|
|
46
64
|
|
|
47
65
|
def __str__(self) -> str:
|
|
48
66
|
return self._app_uri
|
|
@@ -71,7 +89,7 @@ class BioLibApp:
|
|
|
71
89
|
result_prefix: Optional[str] = None,
|
|
72
90
|
timeout: Optional[int] = None,
|
|
73
91
|
notify: bool = False,
|
|
74
|
-
|
|
92
|
+
max_workers: Optional[int] = None,
|
|
75
93
|
experiment: Optional[str] = None,
|
|
76
94
|
temporary_client_secrets: Optional[Dict[str, str]] = None,
|
|
77
95
|
check: bool = False,
|
|
@@ -84,22 +102,18 @@ class BioLibApp:
|
|
|
84
102
|
raise ValueError('The argument "check" cannot be True when blocking is False')
|
|
85
103
|
|
|
86
104
|
if not experiment_id:
|
|
87
|
-
|
|
105
|
+
experiment_to_use = experiment if experiment is not None else self._experiment
|
|
106
|
+
experiment_instance: Optional[Experiment]
|
|
107
|
+
if experiment_to_use:
|
|
108
|
+
experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
|
|
109
|
+
else:
|
|
110
|
+
experiment_instance = Experiment.get_experiment_in_context()
|
|
88
111
|
experiment_id = experiment_instance.uuid if experiment_instance else None
|
|
89
112
|
|
|
90
113
|
module_input_serialized = self._get_serialized_module_input(args, stdin, files)
|
|
91
114
|
|
|
92
115
|
if machine == 'local':
|
|
93
|
-
|
|
94
|
-
raise BioLibError('The argument "blocking" cannot be False when running locally')
|
|
95
|
-
|
|
96
|
-
if experiment_id:
|
|
97
|
-
logger.warning('The argument "experiment_id" is ignored when running locally')
|
|
98
|
-
|
|
99
|
-
if result_prefix:
|
|
100
|
-
logger.warning('The argument "result_prefix" is ignored when running locally')
|
|
101
|
-
|
|
102
|
-
return self._run_locally(module_input_serialized)
|
|
116
|
+
raise BioLibError('Running applications locally with machine="local" is no longer supported.')
|
|
103
117
|
|
|
104
118
|
job = Result._start_job_in_cloud( # pylint: disable=protected-access
|
|
105
119
|
app_uri=self._app_uri,
|
|
@@ -111,11 +125,12 @@ class BioLibApp:
|
|
|
111
125
|
override_command=override_command,
|
|
112
126
|
result_prefix=result_prefix,
|
|
113
127
|
timeout=timeout,
|
|
114
|
-
requested_machine_count=
|
|
128
|
+
requested_machine_count=max_workers,
|
|
115
129
|
temporary_client_secrets=temporary_client_secrets,
|
|
116
130
|
api_client=self._api_client,
|
|
117
131
|
)
|
|
118
|
-
|
|
132
|
+
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
133
|
+
logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
|
|
119
134
|
if blocking:
|
|
120
135
|
# TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
|
|
121
136
|
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
@@ -152,6 +167,8 @@ Example: "app.cli('--help')"
|
|
|
152
167
|
def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
|
|
153
168
|
if args is None:
|
|
154
169
|
args = []
|
|
170
|
+
else:
|
|
171
|
+
args = copy.copy(args)
|
|
155
172
|
|
|
156
173
|
if stdin is None:
|
|
157
174
|
stdin = b''
|
|
@@ -173,18 +190,21 @@ Example: "app.cli('--help')"
|
|
|
173
190
|
for file_path in files:
|
|
174
191
|
path = Path(file_path)
|
|
175
192
|
if path.is_dir():
|
|
193
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
176
194
|
for filename in path.rglob('*'):
|
|
177
195
|
if filename.is_dir():
|
|
178
196
|
continue
|
|
179
197
|
with open(filename, 'rb') as f:
|
|
180
|
-
|
|
181
|
-
files_dict[
|
|
198
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
199
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
182
200
|
else:
|
|
183
201
|
with open(path, 'rb') as f:
|
|
184
202
|
files_dict[path_to_renamed_path(str(path))] = f.read()
|
|
185
203
|
elif isinstance(files, dict):
|
|
186
204
|
files_dict = {}
|
|
187
205
|
for key, value in files.items():
|
|
206
|
+
if '//' in key:
|
|
207
|
+
raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
|
|
188
208
|
if not key.startswith('/'):
|
|
189
209
|
key = '/' + key
|
|
190
210
|
files_dict[key] = value
|
|
@@ -199,11 +219,13 @@ Example: "app.cli('--help')"
|
|
|
199
219
|
files_dict[path_to_renamed_path(arg)] = f.read()
|
|
200
220
|
elif os.path.isdir(arg):
|
|
201
221
|
path = Path(arg)
|
|
222
|
+
renamed_dir = path_to_renamed_path(arg)
|
|
202
223
|
for filename in path.rglob('*'):
|
|
203
224
|
if filename.is_dir():
|
|
204
225
|
continue
|
|
205
226
|
with open(filename, 'rb') as f:
|
|
206
|
-
|
|
227
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
228
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
207
229
|
args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
|
|
208
230
|
|
|
209
231
|
# support --myarg=file.txt
|
|
@@ -214,20 +236,22 @@ Example: "app.cli('--help')"
|
|
|
214
236
|
files_dict[path_to_renamed_path(file_path)] = f.read()
|
|
215
237
|
elif os.path.isdir(file_path):
|
|
216
238
|
path = Path(file_path)
|
|
239
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
217
240
|
for filename in path.rglob('*'):
|
|
218
241
|
if filename.is_dir():
|
|
219
242
|
continue
|
|
220
243
|
with open(filename, 'rb') as f:
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
path_to_renamed_path(file_path, prefix_with_slash=False)
|
|
225
|
-
)
|
|
244
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
245
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
246
|
+
args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
|
|
226
247
|
else:
|
|
227
248
|
pass # a normal string arg was given
|
|
228
249
|
else:
|
|
229
250
|
tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
|
|
230
|
-
if isinstance(arg,
|
|
251
|
+
if isinstance(arg, JsonStringIO):
|
|
252
|
+
file_data = arg.getvalue().encode()
|
|
253
|
+
tmp_filename += '.json'
|
|
254
|
+
elif isinstance(arg, io.StringIO):
|
|
231
255
|
file_data = arg.getvalue().encode()
|
|
232
256
|
elif isinstance(arg, io.BytesIO):
|
|
233
257
|
file_data = arg.getvalue()
|
|
@@ -246,7 +270,7 @@ Example: "app.cli('--help')"
|
|
|
246
270
|
def _run_locally(self, module_input_serialized: bytes) -> Result:
|
|
247
271
|
job_dict = BiolibJobApi.create(
|
|
248
272
|
app_version_id=self._app_version['public_id'],
|
|
249
|
-
app_resource_name_prefix=
|
|
273
|
+
app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
|
|
250
274
|
)
|
|
251
275
|
job = Result(job_dict)
|
|
252
276
|
|
|
@@ -271,7 +295,7 @@ Example: "app.cli('--help')"
|
|
|
271
295
|
continue
|
|
272
296
|
|
|
273
297
|
if isinstance(value, dict):
|
|
274
|
-
value =
|
|
298
|
+
value = JsonStringIO(json.dumps(value))
|
|
275
299
|
elif isinstance(value, (int, float)): # Cast numeric values to strings
|
|
276
300
|
value = str(value)
|
|
277
301
|
|
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
import binascii
|
|
3
|
-
import json
|
|
4
1
|
import os
|
|
5
2
|
from datetime import datetime, timezone
|
|
6
3
|
from json.decoder import JSONDecodeError
|
|
7
4
|
|
|
8
5
|
from biolib._internal.http_client import HttpClient
|
|
6
|
+
from biolib._internal.utils.auth import decode_jwt_without_checking_signature
|
|
9
7
|
from biolib._runtime.runtime import Runtime
|
|
10
8
|
from biolib.biolib_errors import BioLibError
|
|
11
9
|
from biolib.biolib_logging import logger, logger_no_user_data
|
|
12
|
-
from biolib.typing_utils import
|
|
10
|
+
from biolib.typing_utils import Optional, TypedDict
|
|
13
11
|
|
|
14
12
|
from .user_state import UserState
|
|
15
13
|
|
|
@@ -19,10 +17,6 @@ class UserTokens(TypedDict):
|
|
|
19
17
|
refresh: str
|
|
20
18
|
|
|
21
19
|
|
|
22
|
-
class JwtDecodeError(Exception):
|
|
23
|
-
pass
|
|
24
|
-
|
|
25
|
-
|
|
26
20
|
class _ApiClient:
|
|
27
21
|
def __init__(self, base_url: str, access_token: Optional[str] = None):
|
|
28
22
|
self.base_url: str = base_url
|
|
@@ -60,7 +54,7 @@ class _ApiClient:
|
|
|
60
54
|
return
|
|
61
55
|
|
|
62
56
|
if self.access_token:
|
|
63
|
-
decoded_token =
|
|
57
|
+
decoded_token = decode_jwt_without_checking_signature(self.access_token)
|
|
64
58
|
if datetime.now(tz=timezone.utc).timestamp() < decoded_token['payload']['exp'] - 60: # 60 second buffer
|
|
65
59
|
# Token has not expired yet
|
|
66
60
|
return
|
|
@@ -132,44 +126,6 @@ class _ApiClient:
|
|
|
132
126
|
self.access_token = json_response['access_token']
|
|
133
127
|
self.refresh_token = json_response['refresh_token']
|
|
134
128
|
|
|
135
|
-
@staticmethod
|
|
136
|
-
def decode_jwt_without_checking_signature(jwt: str) -> Dict[str, Any]:
|
|
137
|
-
jwt_bytes = jwt.encode('utf-8')
|
|
138
|
-
|
|
139
|
-
try:
|
|
140
|
-
signing_input, _ = jwt_bytes.rsplit(b'.', 1)
|
|
141
|
-
header_segment, payload_segment = signing_input.split(b'.', 1)
|
|
142
|
-
except ValueError as error:
|
|
143
|
-
raise JwtDecodeError('Not enough segments') from error
|
|
144
|
-
|
|
145
|
-
try:
|
|
146
|
-
header_data = base64.urlsafe_b64decode(header_segment)
|
|
147
|
-
except (TypeError, binascii.Error) as error:
|
|
148
|
-
raise JwtDecodeError('Invalid header padding') from error
|
|
149
|
-
|
|
150
|
-
try:
|
|
151
|
-
header = json.loads(header_data)
|
|
152
|
-
except ValueError as error:
|
|
153
|
-
raise JwtDecodeError(f'Invalid header string: {error}') from error
|
|
154
|
-
|
|
155
|
-
if not isinstance(header, dict):
|
|
156
|
-
raise JwtDecodeError('Invalid header string: must be a json object')
|
|
157
|
-
|
|
158
|
-
try:
|
|
159
|
-
payload_data = base64.urlsafe_b64decode(payload_segment)
|
|
160
|
-
except (TypeError, binascii.Error) as error:
|
|
161
|
-
raise JwtDecodeError('Invalid payload padding') from error
|
|
162
|
-
|
|
163
|
-
try:
|
|
164
|
-
payload = json.loads(payload_data)
|
|
165
|
-
except ValueError as error:
|
|
166
|
-
raise JwtDecodeError(f'Invalid payload string: {error}') from error
|
|
167
|
-
|
|
168
|
-
if not isinstance(header, dict):
|
|
169
|
-
raise JwtDecodeError('Invalid payload string: must be a json object')
|
|
170
|
-
|
|
171
|
-
return dict(header=header, payload=payload)
|
|
172
|
-
|
|
173
129
|
|
|
174
130
|
class BiolibApiClient:
|
|
175
131
|
api_client: Optional[_ApiClient] = None
|
|
@@ -68,11 +68,6 @@ class LargeFileSystemMapping(TypedDict):
|
|
|
68
68
|
uuid: str
|
|
69
69
|
|
|
70
70
|
|
|
71
|
-
class PortMapping(TypedDict):
|
|
72
|
-
from_port: int
|
|
73
|
-
to_port: int
|
|
74
|
-
|
|
75
|
-
|
|
76
71
|
class _Module(TypedDict):
|
|
77
72
|
command: str
|
|
78
73
|
environment: Literal['biolib-app', 'biolib-custom', 'biolib-ecr']
|
|
@@ -83,7 +78,7 @@ class _Module(TypedDict):
|
|
|
83
78
|
large_file_systems: List[LargeFileSystemMapping]
|
|
84
79
|
name: str
|
|
85
80
|
output_files_mappings: List[FilesMapping]
|
|
86
|
-
|
|
81
|
+
ports: List[int]
|
|
87
82
|
source_files_mappings: List[FilesMapping]
|
|
88
83
|
working_directory: str
|
|
89
84
|
|
|
@@ -57,6 +57,22 @@ def _get_git_branch_name() -> str:
|
|
|
57
57
|
return ''
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
def _get_git_commit_hash() -> str:
|
|
61
|
+
try:
|
|
62
|
+
github_actions_commit_hash = os.getenv('GITHUB_SHA')
|
|
63
|
+
if github_actions_commit_hash:
|
|
64
|
+
return github_actions_commit_hash
|
|
65
|
+
|
|
66
|
+
gitlab_ci_commit_hash = os.getenv('CI_COMMIT_SHA')
|
|
67
|
+
if gitlab_ci_commit_hash:
|
|
68
|
+
return gitlab_ci_commit_hash
|
|
69
|
+
|
|
70
|
+
result = subprocess.run(['git', 'rev-parse', 'HEAD'], check=True, stdout=subprocess.PIPE, text=True)
|
|
71
|
+
return result.stdout.strip()
|
|
72
|
+
except BaseException:
|
|
73
|
+
return ''
|
|
74
|
+
|
|
75
|
+
|
|
60
76
|
def _get_git_repository_url() -> str:
|
|
61
77
|
try:
|
|
62
78
|
result = subprocess.run(['git', 'remote', 'get-url', 'origin'], check=True, stdout=subprocess.PIPE, text=True)
|
|
@@ -125,6 +141,7 @@ class BiolibAppApi:
|
|
|
125
141
|
'state': 'published',
|
|
126
142
|
'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
|
|
127
143
|
'git_branch_name': _get_git_branch_name(),
|
|
144
|
+
'git_commit_hash': _get_git_commit_hash(),
|
|
128
145
|
'git_repository_url': _get_git_repository_url(),
|
|
129
146
|
}
|
|
130
147
|
if semantic_version:
|