pybiolib 1.2.911__py3-none-any.whl → 1.2.1642__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pybiolib might be problematic. Click here for more details.
- biolib/__init__.py +33 -10
- biolib/_data_record/data_record.py +24 -11
- biolib/_index/index.py +51 -0
- biolib/_index/types.py +7 -0
- biolib/_internal/add_copilot_prompts.py +3 -5
- biolib/_internal/add_gui_files.py +59 -0
- biolib/_internal/data_record/data_record.py +1 -1
- biolib/_internal/data_record/push_data.py +1 -1
- biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
- biolib/_internal/file_utils.py +48 -0
- biolib/_internal/index/__init__.py +1 -0
- biolib/_internal/index/index.py +18 -0
- biolib/_internal/lfs/cache.py +4 -2
- biolib/_internal/push_application.py +89 -23
- biolib/_internal/runtime.py +2 -0
- biolib/_internal/string_utils.py +13 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
- biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
- biolib/_internal/templates/gui_template/App.tsx +53 -0
- biolib/_internal/templates/gui_template/Dockerfile +28 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/index.css +5 -0
- biolib/_internal/templates/gui_template/index.html +13 -0
- biolib/_internal/templates/gui_template/index.tsx +10 -0
- biolib/_internal/templates/gui_template/package.json +27 -0
- biolib/_internal/templates/gui_template/tsconfig.json +24 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
- biolib/_internal/templates/gui_template/vite.config.mts +9 -0
- biolib/_internal/templates/init_template/.biolib/config.yml +1 -0
- biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
- biolib/_internal/templates/init_template/Dockerfile +2 -0
- biolib/_internal/templates/init_template/run.sh +1 -0
- biolib/_internal/templates/templates.py +9 -1
- biolib/_internal/utils/__init__.py +25 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_internal/utils/multinode.py +12 -14
- biolib/_runtime/runtime.py +15 -2
- biolib/_session/session.py +7 -5
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +69 -0
- biolib/_shared/types/account.py +12 -0
- biolib/_shared/types/account_member.py +8 -0
- biolib/{_internal → _shared}/types/experiment.py +1 -0
- biolib/_shared/types/resource.py +17 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/{_internal → _shared}/types/resource_permission.py +1 -1
- biolib/{_internal → _shared}/types/user.py +5 -5
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/client.py +1 -1
- biolib/app/app.py +96 -45
- biolib/biolib_api_client/app_types.py +1 -0
- biolib/biolib_api_client/biolib_app_api.py +26 -0
- biolib/biolib_binary_format/module_input.py +8 -0
- biolib/biolib_binary_format/remote_endpoints.py +3 -3
- biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
- biolib/biolib_logging.py +1 -1
- biolib/cli/__init__.py +2 -1
- biolib/cli/auth.py +4 -16
- biolib/cli/data_record.py +17 -0
- biolib/cli/index.py +32 -0
- biolib/cli/init.py +93 -11
- biolib/cli/lfs.py +1 -1
- biolib/cli/run.py +1 -1
- biolib/cli/start.py +14 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
- biolib/compute_node/job_worker/executors/docker_types.py +1 -1
- biolib/compute_node/job_worker/executors/types.py +6 -5
- biolib/compute_node/job_worker/job_storage.py +2 -1
- biolib/compute_node/job_worker/job_worker.py +155 -90
- biolib/compute_node/job_worker/large_file_system.py +2 -6
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
- biolib/compute_node/remote_host_proxy.py +135 -67
- biolib/compute_node/utils.py +2 -0
- biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +64 -19
- biolib/experiments/experiment.py +98 -16
- biolib/jobs/job.py +128 -31
- biolib/jobs/job_result.py +73 -33
- biolib/jobs/types.py +1 -0
- biolib/sdk/__init__.py +17 -2
- biolib/typing_utils.py +1 -1
- biolib/utils/cache_state.py +2 -2
- biolib/utils/seq_util.py +1 -1
- {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/METADATA +4 -2
- pybiolib-1.2.1642.dist-info/RECORD +180 -0
- {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/WHEEL +1 -1
- biolib/_internal/llm_instructions/.github/instructions/style-react-ts.instructions.md +0 -22
- biolib/_internal/types/__init__.py +0 -6
- biolib/_internal/types/account.py +0 -10
- biolib/utils/app_uri.py +0 -57
- pybiolib-1.2.911.dist-info/RECORD +0 -150
- /biolib/{_internal/llm_instructions → _index}/__init__.py +0 -0
- /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/general-app-knowledge.instructions.md +0 -0
- /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/style-general.instructions.md +0 -0
- /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/style-python.instructions.md +0 -0
- /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/prompts/biolib_app_inputs.prompt.md +0 -0
- /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/prompts/biolib_run_apps.prompt.md +0 -0
- /biolib/{_internal → _shared}/types/app.py +0 -0
- /biolib/{_internal → _shared}/types/data_record.py +0 -0
- /biolib/{_internal → _shared}/types/file_node.py +0 -0
- /biolib/{_internal → _shared}/types/push.py +0 -0
- /biolib/{_internal/types/resource.py → _shared/types/resource_types.py} +0 -0
- /biolib/{_internal → _shared}/types/resource_version.py +0 -0
- /biolib/{_internal → _shared}/types/result.py +0 -0
- /biolib/{_internal → _shared}/types/typing.py +0 -0
- {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/entry_points.txt +0 -0
- {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from urllib.parse import urlparse
|
|
3
|
+
|
|
4
|
+
import biolib.utils
|
|
5
|
+
from biolib.typing_utils import Optional, Tuple
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def parse_result_id_or_url(result_id_or_url: str, default_token: Optional[str] = None) -> Tuple[str, Optional[str]]:
|
|
9
|
+
result_id_or_url = result_id_or_url.strip()
|
|
10
|
+
|
|
11
|
+
if '/' not in result_id_or_url:
|
|
12
|
+
return (result_id_or_url, default_token)
|
|
13
|
+
|
|
14
|
+
if not result_id_or_url.startswith('http://') and not result_id_or_url.startswith('https://'):
|
|
15
|
+
result_id_or_url = 'https://' + result_id_or_url
|
|
16
|
+
|
|
17
|
+
parsed_url = urlparse(result_id_or_url)
|
|
18
|
+
|
|
19
|
+
if biolib.utils.BIOLIB_BASE_URL:
|
|
20
|
+
expected_base = urlparse(biolib.utils.BIOLIB_BASE_URL)
|
|
21
|
+
if parsed_url.scheme != expected_base.scheme or parsed_url.netloc != expected_base.netloc:
|
|
22
|
+
raise ValueError(f'URL must start with {biolib.utils.BIOLIB_BASE_URL}, got: {result_id_or_url}')
|
|
23
|
+
|
|
24
|
+
pattern = r'/results?/(?P<uuid>[a-f0-9-]+)/?(?:\?token=(?P<token>[^&]+))?'
|
|
25
|
+
match = re.search(pattern, result_id_or_url, re.IGNORECASE)
|
|
26
|
+
|
|
27
|
+
if not match:
|
|
28
|
+
raise ValueError(f'URL must be in format <base_url>/results/<UUID>/?token=<token>, got: {result_id_or_url}')
|
|
29
|
+
|
|
30
|
+
uuid = match.group('uuid')
|
|
31
|
+
token = match.group('token') or default_token
|
|
32
|
+
|
|
33
|
+
return (uuid, token)
|
|
@@ -86,25 +86,23 @@ def fasta_batch_records(records, work_per_batch_min, work_per_residue=1, verbose
|
|
|
86
86
|
|
|
87
87
|
batches = []
|
|
88
88
|
batch = []
|
|
89
|
-
|
|
90
|
-
total_work_units = 0
|
|
89
|
+
current_longest_seq_len = 0
|
|
91
90
|
for record in records:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
# Calculate work units
|
|
96
|
-
seq = record.sequence
|
|
97
|
-
sequence_work_units = len(seq) * work_per_residue
|
|
91
|
+
seq_len = len(record.sequence)
|
|
92
|
+
potential_longest_seq_len = max(current_longest_seq_len, seq_len)
|
|
98
93
|
|
|
99
|
-
#
|
|
100
|
-
|
|
101
|
-
total_work_units += sequence_work_units
|
|
94
|
+
# Calculate work units if we were to add this record
|
|
95
|
+
potential_work_units = potential_longest_seq_len * work_per_residue * (len(batch) + 1)
|
|
102
96
|
|
|
103
|
-
|
|
104
|
-
if current_work_units >= work_per_batch_min:
|
|
97
|
+
if potential_work_units >= work_per_batch_min and len(batch) > 0:
|
|
105
98
|
batches.append(batch)
|
|
106
99
|
batch = []
|
|
107
|
-
|
|
100
|
+
current_longest_seq_len = 0
|
|
101
|
+
potential_longest_seq_len = seq_len
|
|
102
|
+
|
|
103
|
+
# Add to batch
|
|
104
|
+
batch.append(record)
|
|
105
|
+
current_longest_seq_len = potential_longest_seq_len
|
|
108
106
|
|
|
109
107
|
# Append last batch if present
|
|
110
108
|
if batch:
|
biolib/_runtime/runtime.py
CHANGED
|
@@ -30,13 +30,26 @@ class Runtime:
|
|
|
30
30
|
return Runtime._get_job_data()['job_auth_token']
|
|
31
31
|
|
|
32
32
|
@staticmethod
|
|
33
|
-
def get_job_requested_machine() -> str:
|
|
34
|
-
|
|
33
|
+
def get_job_requested_machine() -> Optional[str]:
|
|
34
|
+
job_data = Runtime._get_job_data()
|
|
35
|
+
job_requested_machine = job_data.get('job_requested_machine')
|
|
36
|
+
if not job_requested_machine:
|
|
37
|
+
return None
|
|
38
|
+
return job_requested_machine
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def get_job_requested_spot_machine() -> bool:
|
|
42
|
+
job_data = Runtime._get_job_data()
|
|
43
|
+
return job_data.get('job_requested_machine_spot', False)
|
|
35
44
|
|
|
36
45
|
@staticmethod
|
|
37
46
|
def get_app_uri() -> str:
|
|
38
47
|
return Runtime._get_job_data()['app_uri']
|
|
39
48
|
|
|
49
|
+
@staticmethod
|
|
50
|
+
def get_max_workers() -> int:
|
|
51
|
+
return Runtime._get_job_data()['job_reserved_machines']
|
|
52
|
+
|
|
40
53
|
@staticmethod
|
|
41
54
|
def get_secret(secret_name: str) -> bytes:
|
|
42
55
|
assert re.match(
|
biolib/_session/session.py
CHANGED
|
@@ -1,21 +1,23 @@
|
|
|
1
1
|
from biolib import utils
|
|
2
|
-
from biolib.
|
|
2
|
+
from biolib.typing_utils import Optional
|
|
3
3
|
from biolib.api.client import ApiClient, ApiClientInitDict
|
|
4
4
|
from biolib.app import BioLibApp
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class Session:
|
|
8
|
-
def __init__(self, _init_dict: ApiClientInitDict) -> None:
|
|
8
|
+
def __init__(self, _init_dict: ApiClientInitDict, _experiment: Optional[str] = None) -> None:
|
|
9
9
|
self._api = ApiClient(_init_dict=_init_dict)
|
|
10
|
+
self._experiment = _experiment
|
|
10
11
|
|
|
11
12
|
@staticmethod
|
|
12
|
-
def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None) -> 'Session':
|
|
13
|
+
def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None, experiment: Optional[str] = None) -> 'Session':
|
|
13
14
|
return Session(
|
|
14
15
|
_init_dict=ApiClientInitDict(
|
|
15
16
|
refresh_token=refresh_token,
|
|
16
17
|
base_url=base_url or utils.load_base_url_from_env(),
|
|
17
18
|
client_type=client_type,
|
|
18
|
-
)
|
|
19
|
+
),
|
|
20
|
+
_experiment=experiment,
|
|
19
21
|
)
|
|
20
22
|
|
|
21
23
|
def load(self, uri: str, suppress_version_warning: bool = False) -> BioLibApp:
|
|
@@ -39,4 +41,4 @@ class Session:
|
|
|
39
41
|
>>> app = biolib.load('https://biolib.com/biolib/myapp/')
|
|
40
42
|
>>> result = app.cli('--help')
|
|
41
43
|
"""
|
|
42
|
-
return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning)
|
|
44
|
+
return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning, _experiment=self._experiment)
|
|
File without changes
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from .account import AccountDict, AccountDetailedDict
|
|
2
|
+
from .account_member import AccountMemberDict
|
|
3
|
+
from .app import AppDetailedDict, AppSlimDict
|
|
4
|
+
from .data_record import (
|
|
5
|
+
DataRecordDetailedDict,
|
|
6
|
+
DataRecordSlimDict,
|
|
7
|
+
DataRecordTypeDict,
|
|
8
|
+
DataRecordValidationRuleDict,
|
|
9
|
+
SqliteV1Column,
|
|
10
|
+
SqliteV1DatabaseSchema,
|
|
11
|
+
SqliteV1ForeignKey,
|
|
12
|
+
SqliteV1Table,
|
|
13
|
+
)
|
|
14
|
+
from .experiment import (
|
|
15
|
+
DeprecatedExperimentDict,
|
|
16
|
+
ExperimentDetailedDict,
|
|
17
|
+
ExperimentDict,
|
|
18
|
+
ResultCounts,
|
|
19
|
+
)
|
|
20
|
+
from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
|
|
21
|
+
from .push import PushResponseDict
|
|
22
|
+
from .resource import ResourceUriDict, SemanticVersionDict
|
|
23
|
+
from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
|
|
24
|
+
from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
|
|
25
|
+
from .resource_types import ResourceDetailedDict, ResourceDict
|
|
26
|
+
from .resource_version import ResourceVersionDetailedDict, ResourceVersionDict
|
|
27
|
+
from .result import ResultDetailedDict, ResultDict
|
|
28
|
+
from .typing import Optional
|
|
29
|
+
from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
'AccountDetailedDict',
|
|
33
|
+
'AccountDict',
|
|
34
|
+
'AccountMemberDict',
|
|
35
|
+
'AppDetailedDict',
|
|
36
|
+
'AppSlimDict',
|
|
37
|
+
'DataRecordDetailedDict',
|
|
38
|
+
'DataRecordSlimDict',
|
|
39
|
+
'DataRecordTypeDict',
|
|
40
|
+
'DataRecordValidationRuleDict',
|
|
41
|
+
'DeprecatedExperimentDict',
|
|
42
|
+
'EnterpriseSettingsDict',
|
|
43
|
+
'ExperimentDetailedDict',
|
|
44
|
+
'ExperimentDict',
|
|
45
|
+
'FileNodeDict',
|
|
46
|
+
'FileZipMetadataDict',
|
|
47
|
+
'Optional',
|
|
48
|
+
'PushResponseDict',
|
|
49
|
+
'ResourceDeployKeyDict',
|
|
50
|
+
'ResourceDeployKeyWithSecretDict',
|
|
51
|
+
'ResourceDetailedDict',
|
|
52
|
+
'ResourceDict',
|
|
53
|
+
'ResourcePermissionDetailedDict',
|
|
54
|
+
'ResourcePermissionDict',
|
|
55
|
+
'ResourceUriDict',
|
|
56
|
+
'ResourceVersionDetailedDict',
|
|
57
|
+
'ResourceVersionDict',
|
|
58
|
+
'ResultCounts',
|
|
59
|
+
'ResultDetailedDict',
|
|
60
|
+
'ResultDict',
|
|
61
|
+
'SemanticVersionDict',
|
|
62
|
+
'SqliteV1Column',
|
|
63
|
+
'SqliteV1DatabaseSchema',
|
|
64
|
+
'SqliteV1ForeignKey',
|
|
65
|
+
'SqliteV1Table',
|
|
66
|
+
'UserDetailedDict',
|
|
67
|
+
'UserDict',
|
|
68
|
+
'ZipFileNodeDict',
|
|
69
|
+
]
|
|
@@ -20,6 +20,7 @@ class DeprecatedExperimentDict(TypedDict):
|
|
|
20
20
|
class ExperimentDict(DeprecatedExperimentDict):
|
|
21
21
|
uuid: Optional[str]
|
|
22
22
|
name: Optional[str]
|
|
23
|
+
account_uuid: Optional[str]
|
|
23
24
|
created_at: Optional[str]
|
|
24
25
|
finished_at: Optional[str]
|
|
25
26
|
last_created_at: Optional[str]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from .typing import Optional, TypedDict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SemanticVersionDict(TypedDict):
|
|
5
|
+
major: int
|
|
6
|
+
minor: int
|
|
7
|
+
patch: int
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ResourceUriDict(TypedDict):
|
|
11
|
+
account_handle_normalized: str
|
|
12
|
+
account_handle: str
|
|
13
|
+
resource_name_normalized: Optional[str]
|
|
14
|
+
resource_name: Optional[str]
|
|
15
|
+
resource_prefix: Optional[str]
|
|
16
|
+
version: Optional[SemanticVersionDict]
|
|
17
|
+
tag: Optional[str]
|
|
@@ -3,6 +3,7 @@ from .typing import Optional, TypedDict
|
|
|
3
3
|
|
|
4
4
|
|
|
5
5
|
class EnterpriseSettingsDict(TypedDict):
|
|
6
|
+
account_uuid: str
|
|
6
7
|
dashboard_message: Optional[str]
|
|
7
8
|
docs_message: Optional[str]
|
|
8
9
|
featured_dashboard_app_version_uuid: Optional[str]
|
|
@@ -10,10 +11,9 @@ class EnterpriseSettingsDict(TypedDict):
|
|
|
10
11
|
|
|
11
12
|
class UserDict(TypedDict):
|
|
12
13
|
uuid: str
|
|
13
|
-
|
|
14
|
-
enterprise_settings: Optional[EnterpriseSettingsDict]
|
|
15
|
-
intrinsic_account: AccountDict
|
|
14
|
+
account: AccountDict
|
|
16
15
|
|
|
17
16
|
|
|
18
|
-
class UserDetailedDict(
|
|
19
|
-
|
|
17
|
+
class UserDetailedDict(UserDict):
|
|
18
|
+
email: str
|
|
19
|
+
enterprise_settings: Optional[EnterpriseSettingsDict]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from biolib._shared.types import Optional
|
|
4
|
+
from biolib._shared.types.resource import ResourceUriDict, SemanticVersionDict
|
|
5
|
+
from biolib.biolib_errors import BioLibError
|
|
6
|
+
|
|
7
|
+
URI_REGEX = re.compile(
|
|
8
|
+
r'^(@(?P<resource_prefix>[\w._-]+)/)?'
|
|
9
|
+
r'(?P<account_handle>[\w-]+)'
|
|
10
|
+
r'(/(?P<resource_name>[\w-]+))?'
|
|
11
|
+
r'(?::(?P<suffix>[^:]+))?$'
|
|
12
|
+
)
|
|
13
|
+
SEMVER_REGEX = re.compile(r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
|
|
14
|
+
TAG_REGEX = re.compile(r'^[a-z0-9-]{1,128}$')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def normalize_resource_name(string: str) -> str:
|
|
18
|
+
return string.replace('-', '_').lower()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_semantic_version(semantic_version: str) -> SemanticVersionDict:
|
|
22
|
+
if match := SEMVER_REGEX.fullmatch(semantic_version):
|
|
23
|
+
return SemanticVersionDict(
|
|
24
|
+
major=int(match.group('major')),
|
|
25
|
+
minor=int(match.group('minor')),
|
|
26
|
+
patch=int(match.group('patch')),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
raise ValueError('The version must be a valid semantic version in the format of major.minor.patch (1.2.3).')
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_resource_uri(uri: str, use_account_as_name_default: bool = True) -> ResourceUriDict:
|
|
33
|
+
matches = URI_REGEX.match(uri)
|
|
34
|
+
if matches is None:
|
|
35
|
+
raise BioLibError(f"Could not parse resource uri '{uri}', uri did not match regex")
|
|
36
|
+
|
|
37
|
+
version: Optional[SemanticVersionDict] = None
|
|
38
|
+
tag: Optional[str] = None
|
|
39
|
+
|
|
40
|
+
suffix = matches.group('suffix')
|
|
41
|
+
if suffix and suffix != '*':
|
|
42
|
+
try:
|
|
43
|
+
version = parse_semantic_version(suffix)
|
|
44
|
+
except ValueError:
|
|
45
|
+
if TAG_REGEX.fullmatch(suffix):
|
|
46
|
+
tag = suffix
|
|
47
|
+
else:
|
|
48
|
+
raise BioLibError(
|
|
49
|
+
f'Invalid version or tag "{suffix}". '
|
|
50
|
+
'Versions must be semantic versions like "1.2.3". '
|
|
51
|
+
'Tags must be lowercase alphanumeric or dashes and at most 128 characters.'
|
|
52
|
+
) from None
|
|
53
|
+
|
|
54
|
+
resource_prefix_raw: Optional[str] = matches.group('resource_prefix')
|
|
55
|
+
resource_prefix = resource_prefix_raw.lower() if resource_prefix_raw is not None else None
|
|
56
|
+
account_handle: str = matches.group('account_handle')
|
|
57
|
+
account_handle_normalized: str = normalize_resource_name(account_handle)
|
|
58
|
+
resource_name: Optional[str] = matches.group('resource_name')
|
|
59
|
+
|
|
60
|
+
if resource_name:
|
|
61
|
+
resource_name_normalized = normalize_resource_name(resource_name)
|
|
62
|
+
elif use_account_as_name_default:
|
|
63
|
+
resource_name_normalized = account_handle_normalized
|
|
64
|
+
else:
|
|
65
|
+
resource_name_normalized = None
|
|
66
|
+
|
|
67
|
+
return ResourceUriDict(
|
|
68
|
+
resource_prefix=resource_prefix,
|
|
69
|
+
account_handle=account_handle,
|
|
70
|
+
account_handle_normalized=account_handle_normalized,
|
|
71
|
+
resource_name_normalized=resource_name_normalized,
|
|
72
|
+
resource_name=resource_name if resource_name is not None or not use_account_as_name_default else account_handle,
|
|
73
|
+
version=version,
|
|
74
|
+
tag=tag,
|
|
75
|
+
)
|
biolib/api/client.py
CHANGED
|
@@ -8,7 +8,7 @@ from urllib.parse import urlencode, urljoin
|
|
|
8
8
|
import importlib_metadata
|
|
9
9
|
|
|
10
10
|
from biolib._internal.http_client import HttpClient, HttpResponse
|
|
11
|
-
from biolib.
|
|
11
|
+
from biolib._shared.types.typing import Any, Dict, Optional, TypedDict, Union, cast
|
|
12
12
|
from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
|
|
13
13
|
from biolib.biolib_errors import BioLibError
|
|
14
14
|
from biolib.biolib_logging import logger
|
biolib/app/app.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
import io
|
|
2
3
|
import json
|
|
3
4
|
import os
|
|
5
|
+
import posixpath
|
|
4
6
|
import random
|
|
5
7
|
import string
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
|
|
8
10
|
from biolib import utils
|
|
11
|
+
from biolib._internal.file_utils import path_to_renamed_path
|
|
12
|
+
from biolib._runtime.runtime import Runtime
|
|
13
|
+
from biolib._shared.utils import parse_resource_uri
|
|
9
14
|
from biolib.api.client import ApiClient
|
|
10
15
|
from biolib.biolib_api_client import JobState
|
|
11
16
|
from biolib.biolib_api_client.app_types import App, AppVersion
|
|
@@ -18,13 +23,24 @@ from biolib.compute_node.job_worker.job_worker import JobWorker
|
|
|
18
23
|
from biolib.experiments.experiment import Experiment
|
|
19
24
|
from biolib.jobs.job import Result
|
|
20
25
|
from biolib.typing_utils import Dict, Optional
|
|
21
|
-
|
|
22
|
-
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class JsonStringIO(io.StringIO):
|
|
29
|
+
pass
|
|
23
30
|
|
|
24
31
|
|
|
25
32
|
class BioLibApp:
|
|
26
|
-
def __init__(
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
uri: str,
|
|
36
|
+
_api_client: Optional[ApiClient] = None,
|
|
37
|
+
suppress_version_warning: bool = False,
|
|
38
|
+
_experiment: Optional[str] = None,
|
|
39
|
+
):
|
|
27
40
|
self._api_client: Optional[ApiClient] = _api_client
|
|
41
|
+
self._experiment = _experiment
|
|
42
|
+
self._input_uri = uri
|
|
43
|
+
self._parsed_input_uri = parse_resource_uri(uri)
|
|
28
44
|
|
|
29
45
|
app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
|
|
30
46
|
self._app: App = app_response['app']
|
|
@@ -32,16 +48,19 @@ class BioLibApp:
|
|
|
32
48
|
self._app_version: AppVersion = app_response['app_version']
|
|
33
49
|
|
|
34
50
|
if not suppress_version_warning:
|
|
35
|
-
|
|
36
|
-
if parsed_uri['version'] is None:
|
|
51
|
+
if self._parsed_input_uri['version'] is None:
|
|
37
52
|
if Runtime.check_is_environment_biolib_app():
|
|
38
53
|
logger.warning(
|
|
39
54
|
f"No version specified in URI '{uri}'. This will use the default version, "
|
|
40
|
-
f
|
|
55
|
+
f'which may change behaviour over time. Consider locking down the exact version, '
|
|
41
56
|
f"e.g. '{uri}:1.2.3'"
|
|
42
57
|
)
|
|
43
58
|
|
|
44
|
-
|
|
59
|
+
if self._parsed_input_uri['tag']:
|
|
60
|
+
semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
|
|
61
|
+
logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
|
|
62
|
+
else:
|
|
63
|
+
logger.info(f'Loaded {self._app_uri}')
|
|
45
64
|
|
|
46
65
|
def __str__(self) -> str:
|
|
47
66
|
return self._app_uri
|
|
@@ -70,7 +89,7 @@ class BioLibApp:
|
|
|
70
89
|
result_prefix: Optional[str] = None,
|
|
71
90
|
timeout: Optional[int] = None,
|
|
72
91
|
notify: bool = False,
|
|
73
|
-
|
|
92
|
+
max_workers: Optional[int] = None,
|
|
74
93
|
experiment: Optional[str] = None,
|
|
75
94
|
temporary_client_secrets: Optional[Dict[str, str]] = None,
|
|
76
95
|
check: bool = False,
|
|
@@ -83,7 +102,12 @@ class BioLibApp:
|
|
|
83
102
|
raise ValueError('The argument "check" cannot be True when blocking is False')
|
|
84
103
|
|
|
85
104
|
if not experiment_id:
|
|
86
|
-
|
|
105
|
+
experiment_to_use = experiment if experiment is not None else self._experiment
|
|
106
|
+
experiment_instance: Optional[Experiment]
|
|
107
|
+
if experiment_to_use:
|
|
108
|
+
experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
|
|
109
|
+
else:
|
|
110
|
+
experiment_instance = Experiment.get_experiment_in_context()
|
|
87
111
|
experiment_id = experiment_instance.uuid if experiment_instance else None
|
|
88
112
|
|
|
89
113
|
module_input_serialized = self._get_serialized_module_input(args, stdin, files)
|
|
@@ -110,11 +134,12 @@ class BioLibApp:
|
|
|
110
134
|
override_command=override_command,
|
|
111
135
|
result_prefix=result_prefix,
|
|
112
136
|
timeout=timeout,
|
|
113
|
-
requested_machine_count=
|
|
137
|
+
requested_machine_count=max_workers,
|
|
114
138
|
temporary_client_secrets=temporary_client_secrets,
|
|
115
139
|
api_client=self._api_client,
|
|
116
140
|
)
|
|
117
|
-
|
|
141
|
+
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
142
|
+
logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
|
|
118
143
|
if blocking:
|
|
119
144
|
# TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
|
|
120
145
|
if utils.IS_RUNNING_IN_NOTEBOOK:
|
|
@@ -151,6 +176,8 @@ Example: "app.cli('--help')"
|
|
|
151
176
|
def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
|
|
152
177
|
if args is None:
|
|
153
178
|
args = []
|
|
179
|
+
else:
|
|
180
|
+
args = copy.copy(args)
|
|
154
181
|
|
|
155
182
|
if stdin is None:
|
|
156
183
|
stdin = b''
|
|
@@ -168,21 +195,72 @@ Example: "app.cli('--help')"
|
|
|
168
195
|
files = []
|
|
169
196
|
|
|
170
197
|
files_dict = {}
|
|
198
|
+
if isinstance(files, list):
|
|
199
|
+
for file_path in files:
|
|
200
|
+
path = Path(file_path)
|
|
201
|
+
if path.is_dir():
|
|
202
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
203
|
+
for filename in path.rglob('*'):
|
|
204
|
+
if filename.is_dir():
|
|
205
|
+
continue
|
|
206
|
+
with open(filename, 'rb') as f:
|
|
207
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
208
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
209
|
+
else:
|
|
210
|
+
with open(path, 'rb') as f:
|
|
211
|
+
files_dict[path_to_renamed_path(str(path))] = f.read()
|
|
212
|
+
elif isinstance(files, dict):
|
|
213
|
+
files_dict = {}
|
|
214
|
+
for key, value in files.items():
|
|
215
|
+
if '//' in key:
|
|
216
|
+
raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
|
|
217
|
+
if not key.startswith('/'):
|
|
218
|
+
key = '/' + key
|
|
219
|
+
files_dict[key] = value
|
|
220
|
+
else:
|
|
221
|
+
raise Exception('The given files input must be list or dict or None')
|
|
222
|
+
|
|
171
223
|
for idx, arg in enumerate(args):
|
|
172
224
|
if isinstance(arg, str):
|
|
173
225
|
if os.path.isfile(arg) or os.path.isdir(arg):
|
|
174
|
-
|
|
175
|
-
|
|
226
|
+
if os.path.isfile(arg):
|
|
227
|
+
with open(arg, 'rb') as f:
|
|
228
|
+
files_dict[path_to_renamed_path(arg)] = f.read()
|
|
229
|
+
elif os.path.isdir(arg):
|
|
230
|
+
path = Path(arg)
|
|
231
|
+
renamed_dir = path_to_renamed_path(arg)
|
|
232
|
+
for filename in path.rglob('*'):
|
|
233
|
+
if filename.is_dir():
|
|
234
|
+
continue
|
|
235
|
+
with open(filename, 'rb') as f:
|
|
236
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
237
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
238
|
+
args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
|
|
176
239
|
|
|
177
240
|
# support --myarg=file.txt
|
|
178
241
|
elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
|
|
179
|
-
|
|
180
|
-
|
|
242
|
+
file_path = arg.split('=')[-1]
|
|
243
|
+
if os.path.isfile(file_path):
|
|
244
|
+
with open(file_path, 'rb') as f:
|
|
245
|
+
files_dict[path_to_renamed_path(file_path)] = f.read()
|
|
246
|
+
elif os.path.isdir(file_path):
|
|
247
|
+
path = Path(file_path)
|
|
248
|
+
renamed_dir = path_to_renamed_path(file_path)
|
|
249
|
+
for filename in path.rglob('*'):
|
|
250
|
+
if filename.is_dir():
|
|
251
|
+
continue
|
|
252
|
+
with open(filename, 'rb') as f:
|
|
253
|
+
relative_to_dir = filename.resolve().relative_to(path.resolve())
|
|
254
|
+
files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
|
|
255
|
+
args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
|
|
181
256
|
else:
|
|
182
257
|
pass # a normal string arg was given
|
|
183
258
|
else:
|
|
184
259
|
tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
|
|
185
|
-
if isinstance(arg,
|
|
260
|
+
if isinstance(arg, JsonStringIO):
|
|
261
|
+
file_data = arg.getvalue().encode()
|
|
262
|
+
tmp_filename += '.json'
|
|
263
|
+
elif isinstance(arg, io.StringIO):
|
|
186
264
|
file_data = arg.getvalue().encode()
|
|
187
265
|
elif isinstance(arg, io.BytesIO):
|
|
188
266
|
file_data = arg.getvalue()
|
|
@@ -191,33 +269,6 @@ Example: "app.cli('--help')"
|
|
|
191
269
|
files_dict[f'/{tmp_filename}'] = file_data
|
|
192
270
|
args[idx] = tmp_filename
|
|
193
271
|
|
|
194
|
-
if isinstance(files, list):
|
|
195
|
-
for file in files:
|
|
196
|
-
path = Path(file).absolute()
|
|
197
|
-
|
|
198
|
-
# Recursively add data from files if dir
|
|
199
|
-
if path.is_dir():
|
|
200
|
-
for filename in path.rglob('*'):
|
|
201
|
-
if filename.is_dir():
|
|
202
|
-
continue
|
|
203
|
-
file = open(filename, 'rb')
|
|
204
|
-
relative_path = '/' + path.name + '/' + '/'.join(filename.relative_to(path).parts)
|
|
205
|
-
files_dict[relative_path] = file.read()
|
|
206
|
-
file.close()
|
|
207
|
-
|
|
208
|
-
# Add file data
|
|
209
|
-
else:
|
|
210
|
-
file = open(path, 'rb')
|
|
211
|
-
path_short = '/' + path.name
|
|
212
|
-
|
|
213
|
-
files_dict[path_short] = file.read()
|
|
214
|
-
file.close()
|
|
215
|
-
|
|
216
|
-
elif isinstance(files, dict):
|
|
217
|
-
files_dict.update(files)
|
|
218
|
-
else:
|
|
219
|
-
raise Exception('The given files input must be list or dict or None')
|
|
220
|
-
|
|
221
272
|
module_input_serialized: bytes = ModuleInput().serialize(
|
|
222
273
|
stdin=stdin,
|
|
223
274
|
arguments=args,
|
|
@@ -228,7 +279,7 @@ Example: "app.cli('--help')"
|
|
|
228
279
|
def _run_locally(self, module_input_serialized: bytes) -> Result:
|
|
229
280
|
job_dict = BiolibJobApi.create(
|
|
230
281
|
app_version_id=self._app_version['public_id'],
|
|
231
|
-
app_resource_name_prefix=
|
|
282
|
+
app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
|
|
232
283
|
)
|
|
233
284
|
job = Result(job_dict)
|
|
234
285
|
|
|
@@ -253,7 +304,7 @@ Example: "app.cli('--help')"
|
|
|
253
304
|
continue
|
|
254
305
|
|
|
255
306
|
if isinstance(value, dict):
|
|
256
|
-
value =
|
|
307
|
+
value = JsonStringIO(json.dumps(value))
|
|
257
308
|
elif isinstance(value, (int, float)): # Cast numeric values to strings
|
|
258
309
|
value = str(value)
|
|
259
310
|
|