pybiolib 1.2.911__py3-none-any.whl → 1.2.1642__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pybiolib might be problematic. Click here for more details.

Files changed (113) hide show
  1. biolib/__init__.py +33 -10
  2. biolib/_data_record/data_record.py +24 -11
  3. biolib/_index/index.py +51 -0
  4. biolib/_index/types.py +7 -0
  5. biolib/_internal/add_copilot_prompts.py +3 -5
  6. biolib/_internal/add_gui_files.py +59 -0
  7. biolib/_internal/data_record/data_record.py +1 -1
  8. biolib/_internal/data_record/push_data.py +1 -1
  9. biolib/_internal/data_record/remote_storage_endpoint.py +3 -3
  10. biolib/_internal/file_utils.py +48 -0
  11. biolib/_internal/index/__init__.py +1 -0
  12. biolib/_internal/index/index.py +18 -0
  13. biolib/_internal/lfs/cache.py +4 -2
  14. biolib/_internal/push_application.py +89 -23
  15. biolib/_internal/runtime.py +2 -0
  16. biolib/_internal/string_utils.py +13 -0
  17. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  18. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  19. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  20. biolib/_internal/templates/gui_template/App.tsx +53 -0
  21. biolib/_internal/templates/gui_template/Dockerfile +28 -0
  22. biolib/_internal/templates/gui_template/biolib-sdk.ts +37 -0
  23. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  24. biolib/_internal/templates/gui_template/index.css +5 -0
  25. biolib/_internal/templates/gui_template/index.html +13 -0
  26. biolib/_internal/templates/gui_template/index.tsx +10 -0
  27. biolib/_internal/templates/gui_template/package.json +27 -0
  28. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  29. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +49 -0
  30. biolib/_internal/templates/gui_template/vite.config.mts +9 -0
  31. biolib/_internal/templates/init_template/.biolib/config.yml +1 -0
  32. biolib/_internal/templates/init_template/.github/workflows/biolib.yml +6 -1
  33. biolib/_internal/templates/init_template/Dockerfile +2 -0
  34. biolib/_internal/templates/init_template/run.sh +1 -0
  35. biolib/_internal/templates/templates.py +9 -1
  36. biolib/_internal/utils/__init__.py +25 -0
  37. biolib/_internal/utils/job_url.py +33 -0
  38. biolib/_internal/utils/multinode.py +12 -14
  39. biolib/_runtime/runtime.py +15 -2
  40. biolib/_session/session.py +7 -5
  41. biolib/_shared/__init__.py +0 -0
  42. biolib/_shared/types/__init__.py +69 -0
  43. biolib/_shared/types/account.py +12 -0
  44. biolib/_shared/types/account_member.py +8 -0
  45. biolib/{_internal → _shared}/types/experiment.py +1 -0
  46. biolib/_shared/types/resource.py +17 -0
  47. biolib/_shared/types/resource_deploy_key.py +11 -0
  48. biolib/{_internal → _shared}/types/resource_permission.py +1 -1
  49. biolib/{_internal → _shared}/types/user.py +5 -5
  50. biolib/_shared/utils/__init__.py +7 -0
  51. biolib/_shared/utils/resource_uri.py +75 -0
  52. biolib/api/client.py +1 -1
  53. biolib/app/app.py +96 -45
  54. biolib/biolib_api_client/app_types.py +1 -0
  55. biolib/biolib_api_client/biolib_app_api.py +26 -0
  56. biolib/biolib_binary_format/module_input.py +8 -0
  57. biolib/biolib_binary_format/remote_endpoints.py +3 -3
  58. biolib/biolib_binary_format/remote_stream_seeker.py +39 -25
  59. biolib/biolib_logging.py +1 -1
  60. biolib/cli/__init__.py +2 -1
  61. biolib/cli/auth.py +4 -16
  62. biolib/cli/data_record.py +17 -0
  63. biolib/cli/index.py +32 -0
  64. biolib/cli/init.py +93 -11
  65. biolib/cli/lfs.py +1 -1
  66. biolib/cli/run.py +1 -1
  67. biolib/cli/start.py +14 -1
  68. biolib/compute_node/job_worker/executors/docker_executor.py +31 -9
  69. biolib/compute_node/job_worker/executors/docker_types.py +1 -1
  70. biolib/compute_node/job_worker/executors/types.py +6 -5
  71. biolib/compute_node/job_worker/job_storage.py +2 -1
  72. biolib/compute_node/job_worker/job_worker.py +155 -90
  73. biolib/compute_node/job_worker/large_file_system.py +2 -6
  74. biolib/compute_node/job_worker/network_alloc.py +99 -0
  75. biolib/compute_node/job_worker/network_buffer.py +240 -0
  76. biolib/compute_node/job_worker/utilization_reporter_thread.py +2 -2
  77. biolib/compute_node/remote_host_proxy.py +135 -67
  78. biolib/compute_node/utils.py +2 -0
  79. biolib/compute_node/webserver/compute_node_results_proxy.py +188 -0
  80. biolib/compute_node/webserver/proxy_utils.py +28 -0
  81. biolib/compute_node/webserver/webserver.py +64 -19
  82. biolib/experiments/experiment.py +98 -16
  83. biolib/jobs/job.py +128 -31
  84. biolib/jobs/job_result.py +73 -33
  85. biolib/jobs/types.py +1 -0
  86. biolib/sdk/__init__.py +17 -2
  87. biolib/typing_utils.py +1 -1
  88. biolib/utils/cache_state.py +2 -2
  89. biolib/utils/seq_util.py +1 -1
  90. {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/METADATA +4 -2
  91. pybiolib-1.2.1642.dist-info/RECORD +180 -0
  92. {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/WHEEL +1 -1
  93. biolib/_internal/llm_instructions/.github/instructions/style-react-ts.instructions.md +0 -22
  94. biolib/_internal/types/__init__.py +0 -6
  95. biolib/_internal/types/account.py +0 -10
  96. biolib/utils/app_uri.py +0 -57
  97. pybiolib-1.2.911.dist-info/RECORD +0 -150
  98. /biolib/{_internal/llm_instructions → _index}/__init__.py +0 -0
  99. /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/general-app-knowledge.instructions.md +0 -0
  100. /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/style-general.instructions.md +0 -0
  101. /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/instructions/style-python.instructions.md +0 -0
  102. /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/prompts/biolib_app_inputs.prompt.md +0 -0
  103. /biolib/_internal/{llm_instructions → templates/copilot_template}/.github/prompts/biolib_run_apps.prompt.md +0 -0
  104. /biolib/{_internal → _shared}/types/app.py +0 -0
  105. /biolib/{_internal → _shared}/types/data_record.py +0 -0
  106. /biolib/{_internal → _shared}/types/file_node.py +0 -0
  107. /biolib/{_internal → _shared}/types/push.py +0 -0
  108. /biolib/{_internal/types/resource.py → _shared/types/resource_types.py} +0 -0
  109. /biolib/{_internal → _shared}/types/resource_version.py +0 -0
  110. /biolib/{_internal → _shared}/types/result.py +0 -0
  111. /biolib/{_internal → _shared}/types/typing.py +0 -0
  112. {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info}/entry_points.txt +0 -0
  113. {pybiolib-1.2.911.dist-info → pybiolib-1.2.1642.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,33 @@
1
+ import re
2
+ from urllib.parse import urlparse
3
+
4
+ import biolib.utils
5
+ from biolib.typing_utils import Optional, Tuple
6
+
7
+
8
+ def parse_result_id_or_url(result_id_or_url: str, default_token: Optional[str] = None) -> Tuple[str, Optional[str]]:
9
+ result_id_or_url = result_id_or_url.strip()
10
+
11
+ if '/' not in result_id_or_url:
12
+ return (result_id_or_url, default_token)
13
+
14
+ if not result_id_or_url.startswith('http://') and not result_id_or_url.startswith('https://'):
15
+ result_id_or_url = 'https://' + result_id_or_url
16
+
17
+ parsed_url = urlparse(result_id_or_url)
18
+
19
+ if biolib.utils.BIOLIB_BASE_URL:
20
+ expected_base = urlparse(biolib.utils.BIOLIB_BASE_URL)
21
+ if parsed_url.scheme != expected_base.scheme or parsed_url.netloc != expected_base.netloc:
22
+ raise ValueError(f'URL must start with {biolib.utils.BIOLIB_BASE_URL}, got: {result_id_or_url}')
23
+
24
+ pattern = r'/results?/(?P<uuid>[a-f0-9-]+)/?(?:\?token=(?P<token>[^&]+))?'
25
+ match = re.search(pattern, result_id_or_url, re.IGNORECASE)
26
+
27
+ if not match:
28
+ raise ValueError(f'URL must be in format <base_url>/results/<UUID>/?token=<token>, got: {result_id_or_url}')
29
+
30
+ uuid = match.group('uuid')
31
+ token = match.group('token') or default_token
32
+
33
+ return (uuid, token)
@@ -86,25 +86,23 @@ def fasta_batch_records(records, work_per_batch_min, work_per_residue=1, verbose
86
86
 
87
87
  batches = []
88
88
  batch = []
89
- current_work_units = 0
90
- total_work_units = 0
89
+ current_longest_seq_len = 0
91
90
  for record in records:
92
- # Add to batch
93
- batch.append(record)
94
-
95
- # Calculate work units
96
- seq = record.sequence
97
- sequence_work_units = len(seq) * work_per_residue
91
+ seq_len = len(record.sequence)
92
+ potential_longest_seq_len = max(current_longest_seq_len, seq_len)
98
93
 
99
- # Increase counters
100
- current_work_units += sequence_work_units
101
- total_work_units += sequence_work_units
94
+ # Calculate work units if we were to add this record
95
+ potential_work_units = potential_longest_seq_len * work_per_residue * (len(batch) + 1)
102
96
 
103
- # If above limit, start a new batch
104
- if current_work_units >= work_per_batch_min:
97
+ if potential_work_units >= work_per_batch_min and len(batch) > 0:
105
98
  batches.append(batch)
106
99
  batch = []
107
- current_work_units = 0
100
+ current_longest_seq_len = 0
101
+ potential_longest_seq_len = seq_len
102
+
103
+ # Add to batch
104
+ batch.append(record)
105
+ current_longest_seq_len = potential_longest_seq_len
108
106
 
109
107
  # Append last batch if present
110
108
  if batch:
@@ -30,13 +30,26 @@ class Runtime:
30
30
  return Runtime._get_job_data()['job_auth_token']
31
31
 
32
32
  @staticmethod
33
- def get_job_requested_machine() -> str:
34
- return Runtime._get_job_data()['job_requested_machine']
33
+ def get_job_requested_machine() -> Optional[str]:
34
+ job_data = Runtime._get_job_data()
35
+ job_requested_machine = job_data.get('job_requested_machine')
36
+ if not job_requested_machine:
37
+ return None
38
+ return job_requested_machine
39
+
40
+ @staticmethod
41
+ def get_job_requested_spot_machine() -> bool:
42
+ job_data = Runtime._get_job_data()
43
+ return job_data.get('job_requested_machine_spot', False)
35
44
 
36
45
  @staticmethod
37
46
  def get_app_uri() -> str:
38
47
  return Runtime._get_job_data()['app_uri']
39
48
 
49
+ @staticmethod
50
+ def get_max_workers() -> int:
51
+ return Runtime._get_job_data()['job_reserved_machines']
52
+
40
53
  @staticmethod
41
54
  def get_secret(secret_name: str) -> bytes:
42
55
  assert re.match(
@@ -1,21 +1,23 @@
1
1
  from biolib import utils
2
- from biolib._internal.types import Optional
2
+ from biolib.typing_utils import Optional
3
3
  from biolib.api.client import ApiClient, ApiClientInitDict
4
4
  from biolib.app import BioLibApp
5
5
 
6
6
 
7
7
  class Session:
8
- def __init__(self, _init_dict: ApiClientInitDict) -> None:
8
+ def __init__(self, _init_dict: ApiClientInitDict, _experiment: Optional[str] = None) -> None:
9
9
  self._api = ApiClient(_init_dict=_init_dict)
10
+ self._experiment = _experiment
10
11
 
11
12
  @staticmethod
12
- def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None) -> 'Session':
13
+ def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None, experiment: Optional[str] = None) -> 'Session':
13
14
  return Session(
14
15
  _init_dict=ApiClientInitDict(
15
16
  refresh_token=refresh_token,
16
17
  base_url=base_url or utils.load_base_url_from_env(),
17
18
  client_type=client_type,
18
- )
19
+ ),
20
+ _experiment=experiment,
19
21
  )
20
22
 
21
23
  def load(self, uri: str, suppress_version_warning: bool = False) -> BioLibApp:
@@ -39,4 +41,4 @@ class Session:
39
41
  >>> app = biolib.load('https://biolib.com/biolib/myapp/')
40
42
  >>> result = app.cli('--help')
41
43
  """
42
- return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning)
44
+ return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning, _experiment=self._experiment)
File without changes
@@ -0,0 +1,69 @@
1
+ from .account import AccountDict, AccountDetailedDict
2
+ from .account_member import AccountMemberDict
3
+ from .app import AppDetailedDict, AppSlimDict
4
+ from .data_record import (
5
+ DataRecordDetailedDict,
6
+ DataRecordSlimDict,
7
+ DataRecordTypeDict,
8
+ DataRecordValidationRuleDict,
9
+ SqliteV1Column,
10
+ SqliteV1DatabaseSchema,
11
+ SqliteV1ForeignKey,
12
+ SqliteV1Table,
13
+ )
14
+ from .experiment import (
15
+ DeprecatedExperimentDict,
16
+ ExperimentDetailedDict,
17
+ ExperimentDict,
18
+ ResultCounts,
19
+ )
20
+ from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
21
+ from .push import PushResponseDict
22
+ from .resource import ResourceUriDict, SemanticVersionDict
23
+ from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
24
+ from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
25
+ from .resource_types import ResourceDetailedDict, ResourceDict
26
+ from .resource_version import ResourceVersionDetailedDict, ResourceVersionDict
27
+ from .result import ResultDetailedDict, ResultDict
28
+ from .typing import Optional
29
+ from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
30
+
31
+ __all__ = [
32
+ 'AccountDetailedDict',
33
+ 'AccountDict',
34
+ 'AccountMemberDict',
35
+ 'AppDetailedDict',
36
+ 'AppSlimDict',
37
+ 'DataRecordDetailedDict',
38
+ 'DataRecordSlimDict',
39
+ 'DataRecordTypeDict',
40
+ 'DataRecordValidationRuleDict',
41
+ 'DeprecatedExperimentDict',
42
+ 'EnterpriseSettingsDict',
43
+ 'ExperimentDetailedDict',
44
+ 'ExperimentDict',
45
+ 'FileNodeDict',
46
+ 'FileZipMetadataDict',
47
+ 'Optional',
48
+ 'PushResponseDict',
49
+ 'ResourceDeployKeyDict',
50
+ 'ResourceDeployKeyWithSecretDict',
51
+ 'ResourceDetailedDict',
52
+ 'ResourceDict',
53
+ 'ResourcePermissionDetailedDict',
54
+ 'ResourcePermissionDict',
55
+ 'ResourceUriDict',
56
+ 'ResourceVersionDetailedDict',
57
+ 'ResourceVersionDict',
58
+ 'ResultCounts',
59
+ 'ResultDetailedDict',
60
+ 'ResultDict',
61
+ 'SemanticVersionDict',
62
+ 'SqliteV1Column',
63
+ 'SqliteV1DatabaseSchema',
64
+ 'SqliteV1ForeignKey',
65
+ 'SqliteV1Table',
66
+ 'UserDetailedDict',
67
+ 'UserDict',
68
+ 'ZipFileNodeDict',
69
+ ]
@@ -0,0 +1,12 @@
1
+ from .typing import TypedDict
2
+
3
+
4
+ class AccountDict(TypedDict):
5
+ uuid: str
6
+ handle: str
7
+ display_name: str
8
+ description: str
9
+
10
+
11
+ class AccountDetailedDict(AccountDict):
12
+ bio: str
@@ -0,0 +1,8 @@
1
+ from .typing import Literal, TypedDict
2
+ from .user import UserDict
3
+
4
+
5
+ class AccountMemberDict(TypedDict):
6
+ user: UserDict
7
+ role: Literal['member', 'admin']
8
+ added_at: str
@@ -20,6 +20,7 @@ class DeprecatedExperimentDict(TypedDict):
20
20
  class ExperimentDict(DeprecatedExperimentDict):
21
21
  uuid: Optional[str]
22
22
  name: Optional[str]
23
+ account_uuid: Optional[str]
23
24
  created_at: Optional[str]
24
25
  finished_at: Optional[str]
25
26
  last_created_at: Optional[str]
@@ -0,0 +1,17 @@
1
+ from .typing import Optional, TypedDict
2
+
3
+
4
+ class SemanticVersionDict(TypedDict):
5
+ major: int
6
+ minor: int
7
+ patch: int
8
+
9
+
10
+ class ResourceUriDict(TypedDict):
11
+ account_handle_normalized: str
12
+ account_handle: str
13
+ resource_name_normalized: Optional[str]
14
+ resource_name: Optional[str]
15
+ resource_prefix: Optional[str]
16
+ version: Optional[SemanticVersionDict]
17
+ tag: Optional[str]
@@ -0,0 +1,11 @@
1
+ from .typing import TypedDict
2
+
3
+
4
+ class ResourceDeployKeyDict(TypedDict):
5
+ uuid: str
6
+ created_at: str
7
+ name: str
8
+
9
+
10
+ class ResourceDeployKeyWithSecretDict(ResourceDeployKeyDict):
11
+ secret_key: str
@@ -1,4 +1,4 @@
1
- from .resource import ResourceDict
1
+ from .resource_types import ResourceDict
2
2
  from .typing import TypedDict
3
3
 
4
4
 
@@ -3,6 +3,7 @@ from .typing import Optional, TypedDict
3
3
 
4
4
 
5
5
  class EnterpriseSettingsDict(TypedDict):
6
+ account_uuid: str
6
7
  dashboard_message: Optional[str]
7
8
  docs_message: Optional[str]
8
9
  featured_dashboard_app_version_uuid: Optional[str]
@@ -10,10 +11,9 @@ class EnterpriseSettingsDict(TypedDict):
10
11
 
11
12
  class UserDict(TypedDict):
12
13
  uuid: str
13
- email: str
14
- enterprise_settings: Optional[EnterpriseSettingsDict]
15
- intrinsic_account: AccountDict
14
+ account: AccountDict
16
15
 
17
16
 
18
- class UserDetailedDict(TypedDict):
19
- pass
17
+ class UserDetailedDict(UserDict):
18
+ email: str
19
+ enterprise_settings: Optional[EnterpriseSettingsDict]
@@ -0,0 +1,7 @@
1
+ from biolib._shared.utils.resource_uri import normalize_resource_name, parse_resource_uri, parse_semantic_version
2
+
3
+ __all__ = [
4
+ 'normalize_resource_name',
5
+ 'parse_resource_uri',
6
+ 'parse_semantic_version',
7
+ ]
@@ -0,0 +1,75 @@
1
+ import re
2
+
3
+ from biolib._shared.types import Optional
4
+ from biolib._shared.types.resource import ResourceUriDict, SemanticVersionDict
5
+ from biolib.biolib_errors import BioLibError
6
+
7
+ URI_REGEX = re.compile(
8
+ r'^(@(?P<resource_prefix>[\w._-]+)/)?'
9
+ r'(?P<account_handle>[\w-]+)'
10
+ r'(/(?P<resource_name>[\w-]+))?'
11
+ r'(?::(?P<suffix>[^:]+))?$'
12
+ )
13
+ SEMVER_REGEX = re.compile(r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)$')
14
+ TAG_REGEX = re.compile(r'^[a-z0-9-]{1,128}$')
15
+
16
+
17
+ def normalize_resource_name(string: str) -> str:
18
+ return string.replace('-', '_').lower()
19
+
20
+
21
+ def parse_semantic_version(semantic_version: str) -> SemanticVersionDict:
22
+ if match := SEMVER_REGEX.fullmatch(semantic_version):
23
+ return SemanticVersionDict(
24
+ major=int(match.group('major')),
25
+ minor=int(match.group('minor')),
26
+ patch=int(match.group('patch')),
27
+ )
28
+
29
+ raise ValueError('The version must be a valid semantic version in the format of major.minor.patch (1.2.3).')
30
+
31
+
32
+ def parse_resource_uri(uri: str, use_account_as_name_default: bool = True) -> ResourceUriDict:
33
+ matches = URI_REGEX.match(uri)
34
+ if matches is None:
35
+ raise BioLibError(f"Could not parse resource uri '{uri}', uri did not match regex")
36
+
37
+ version: Optional[SemanticVersionDict] = None
38
+ tag: Optional[str] = None
39
+
40
+ suffix = matches.group('suffix')
41
+ if suffix and suffix != '*':
42
+ try:
43
+ version = parse_semantic_version(suffix)
44
+ except ValueError:
45
+ if TAG_REGEX.fullmatch(suffix):
46
+ tag = suffix
47
+ else:
48
+ raise BioLibError(
49
+ f'Invalid version or tag "{suffix}". '
50
+ 'Versions must be semantic versions like "1.2.3". '
51
+ 'Tags must be lowercase alphanumeric or dashes and at most 128 characters.'
52
+ ) from None
53
+
54
+ resource_prefix_raw: Optional[str] = matches.group('resource_prefix')
55
+ resource_prefix = resource_prefix_raw.lower() if resource_prefix_raw is not None else None
56
+ account_handle: str = matches.group('account_handle')
57
+ account_handle_normalized: str = normalize_resource_name(account_handle)
58
+ resource_name: Optional[str] = matches.group('resource_name')
59
+
60
+ if resource_name:
61
+ resource_name_normalized = normalize_resource_name(resource_name)
62
+ elif use_account_as_name_default:
63
+ resource_name_normalized = account_handle_normalized
64
+ else:
65
+ resource_name_normalized = None
66
+
67
+ return ResourceUriDict(
68
+ resource_prefix=resource_prefix,
69
+ account_handle=account_handle,
70
+ account_handle_normalized=account_handle_normalized,
71
+ resource_name_normalized=resource_name_normalized,
72
+ resource_name=resource_name if resource_name is not None or not use_account_as_name_default else account_handle,
73
+ version=version,
74
+ tag=tag,
75
+ )
biolib/api/client.py CHANGED
@@ -8,7 +8,7 @@ from urllib.parse import urlencode, urljoin
8
8
  import importlib_metadata
9
9
 
10
10
  from biolib._internal.http_client import HttpClient, HttpResponse
11
- from biolib._internal.types.typing import Any, Dict, Optional, TypedDict, Union, cast
11
+ from biolib._shared.types.typing import Any, Dict, Optional, TypedDict, Union, cast
12
12
  from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
13
13
  from biolib.biolib_errors import BioLibError
14
14
  from biolib.biolib_logging import logger
biolib/app/app.py CHANGED
@@ -1,11 +1,16 @@
1
+ import copy
1
2
  import io
2
3
  import json
3
4
  import os
5
+ import posixpath
4
6
  import random
5
7
  import string
6
8
  from pathlib import Path
7
9
 
8
10
  from biolib import utils
11
+ from biolib._internal.file_utils import path_to_renamed_path
12
+ from biolib._runtime.runtime import Runtime
13
+ from biolib._shared.utils import parse_resource_uri
9
14
  from biolib.api.client import ApiClient
10
15
  from biolib.biolib_api_client import JobState
11
16
  from biolib.biolib_api_client.app_types import App, AppVersion
@@ -18,13 +23,24 @@ from biolib.compute_node.job_worker.job_worker import JobWorker
18
23
  from biolib.experiments.experiment import Experiment
19
24
  from biolib.jobs.job import Result
20
25
  from biolib.typing_utils import Dict, Optional
21
- from biolib.utils.app_uri import parse_app_uri
22
- from biolib._runtime.runtime import Runtime
26
+
27
+
28
+ class JsonStringIO(io.StringIO):
29
+ pass
23
30
 
24
31
 
25
32
  class BioLibApp:
26
- def __init__(self, uri: str, _api_client: Optional[ApiClient] = None, suppress_version_warning: bool = False):
33
+ def __init__(
34
+ self,
35
+ uri: str,
36
+ _api_client: Optional[ApiClient] = None,
37
+ suppress_version_warning: bool = False,
38
+ _experiment: Optional[str] = None,
39
+ ):
27
40
  self._api_client: Optional[ApiClient] = _api_client
41
+ self._experiment = _experiment
42
+ self._input_uri = uri
43
+ self._parsed_input_uri = parse_resource_uri(uri)
28
44
 
29
45
  app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
30
46
  self._app: App = app_response['app']
@@ -32,16 +48,19 @@ class BioLibApp:
32
48
  self._app_version: AppVersion = app_response['app_version']
33
49
 
34
50
  if not suppress_version_warning:
35
- parsed_uri = parse_app_uri(uri)
36
- if parsed_uri['version'] is None:
51
+ if self._parsed_input_uri['version'] is None:
37
52
  if Runtime.check_is_environment_biolib_app():
38
53
  logger.warning(
39
54
  f"No version specified in URI '{uri}'. This will use the default version, "
40
- f"which may change behaviour over time. Consider locking down the exact version, "
55
+ f'which may change behaviour over time. Consider locking down the exact version, '
41
56
  f"e.g. '{uri}:1.2.3'"
42
57
  )
43
58
 
44
- logger.info(f'Loaded project {self._app_uri}')
59
+ if self._parsed_input_uri['tag']:
60
+ semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
61
+ logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
62
+ else:
63
+ logger.info(f'Loaded {self._app_uri}')
45
64
 
46
65
  def __str__(self) -> str:
47
66
  return self._app_uri
@@ -70,7 +89,7 @@ class BioLibApp:
70
89
  result_prefix: Optional[str] = None,
71
90
  timeout: Optional[int] = None,
72
91
  notify: bool = False,
73
- machine_count: Optional[int] = None,
92
+ max_workers: Optional[int] = None,
74
93
  experiment: Optional[str] = None,
75
94
  temporary_client_secrets: Optional[Dict[str, str]] = None,
76
95
  check: bool = False,
@@ -83,7 +102,12 @@ class BioLibApp:
83
102
  raise ValueError('The argument "check" cannot be True when blocking is False')
84
103
 
85
104
  if not experiment_id:
86
- experiment_instance = Experiment(experiment) if experiment else Experiment.get_experiment_in_context()
105
+ experiment_to_use = experiment if experiment is not None else self._experiment
106
+ experiment_instance: Optional[Experiment]
107
+ if experiment_to_use:
108
+ experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
109
+ else:
110
+ experiment_instance = Experiment.get_experiment_in_context()
87
111
  experiment_id = experiment_instance.uuid if experiment_instance else None
88
112
 
89
113
  module_input_serialized = self._get_serialized_module_input(args, stdin, files)
@@ -110,11 +134,12 @@ class BioLibApp:
110
134
  override_command=override_command,
111
135
  result_prefix=result_prefix,
112
136
  timeout=timeout,
113
- requested_machine_count=machine_count,
137
+ requested_machine_count=max_workers,
114
138
  temporary_client_secrets=temporary_client_secrets,
115
139
  api_client=self._api_client,
116
140
  )
117
- logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
141
+ if utils.IS_RUNNING_IN_NOTEBOOK:
142
+ logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
118
143
  if blocking:
119
144
  # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
120
145
  if utils.IS_RUNNING_IN_NOTEBOOK:
@@ -151,6 +176,8 @@ Example: "app.cli('--help')"
151
176
  def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
152
177
  if args is None:
153
178
  args = []
179
+ else:
180
+ args = copy.copy(args)
154
181
 
155
182
  if stdin is None:
156
183
  stdin = b''
@@ -168,21 +195,72 @@ Example: "app.cli('--help')"
168
195
  files = []
169
196
 
170
197
  files_dict = {}
198
+ if isinstance(files, list):
199
+ for file_path in files:
200
+ path = Path(file_path)
201
+ if path.is_dir():
202
+ renamed_dir = path_to_renamed_path(file_path)
203
+ for filename in path.rglob('*'):
204
+ if filename.is_dir():
205
+ continue
206
+ with open(filename, 'rb') as f:
207
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
208
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
209
+ else:
210
+ with open(path, 'rb') as f:
211
+ files_dict[path_to_renamed_path(str(path))] = f.read()
212
+ elif isinstance(files, dict):
213
+ files_dict = {}
214
+ for key, value in files.items():
215
+ if '//' in key:
216
+ raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
217
+ if not key.startswith('/'):
218
+ key = '/' + key
219
+ files_dict[key] = value
220
+ else:
221
+ raise Exception('The given files input must be list or dict or None')
222
+
171
223
  for idx, arg in enumerate(args):
172
224
  if isinstance(arg, str):
173
225
  if os.path.isfile(arg) or os.path.isdir(arg):
174
- files.append(arg)
175
- args[idx] = Path(arg).name
226
+ if os.path.isfile(arg):
227
+ with open(arg, 'rb') as f:
228
+ files_dict[path_to_renamed_path(arg)] = f.read()
229
+ elif os.path.isdir(arg):
230
+ path = Path(arg)
231
+ renamed_dir = path_to_renamed_path(arg)
232
+ for filename in path.rglob('*'):
233
+ if filename.is_dir():
234
+ continue
235
+ with open(filename, 'rb') as f:
236
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
237
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
238
+ args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
176
239
 
177
240
  # support --myarg=file.txt
178
241
  elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
179
- files.append(arg.split('=')[-1])
180
- args[idx] = arg.split('=')[0] + '=' + Path(arg.split('=')[-1]).name
242
+ file_path = arg.split('=')[-1]
243
+ if os.path.isfile(file_path):
244
+ with open(file_path, 'rb') as f:
245
+ files_dict[path_to_renamed_path(file_path)] = f.read()
246
+ elif os.path.isdir(file_path):
247
+ path = Path(file_path)
248
+ renamed_dir = path_to_renamed_path(file_path)
249
+ for filename in path.rglob('*'):
250
+ if filename.is_dir():
251
+ continue
252
+ with open(filename, 'rb') as f:
253
+ relative_to_dir = filename.resolve().relative_to(path.resolve())
254
+ files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
255
+ args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
181
256
  else:
182
257
  pass # a normal string arg was given
183
258
  else:
184
259
  tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
185
- if isinstance(arg, io.StringIO):
260
+ if isinstance(arg, JsonStringIO):
261
+ file_data = arg.getvalue().encode()
262
+ tmp_filename += '.json'
263
+ elif isinstance(arg, io.StringIO):
186
264
  file_data = arg.getvalue().encode()
187
265
  elif isinstance(arg, io.BytesIO):
188
266
  file_data = arg.getvalue()
@@ -191,33 +269,6 @@ Example: "app.cli('--help')"
191
269
  files_dict[f'/{tmp_filename}'] = file_data
192
270
  args[idx] = tmp_filename
193
271
 
194
- if isinstance(files, list):
195
- for file in files:
196
- path = Path(file).absolute()
197
-
198
- # Recursively add data from files if dir
199
- if path.is_dir():
200
- for filename in path.rglob('*'):
201
- if filename.is_dir():
202
- continue
203
- file = open(filename, 'rb')
204
- relative_path = '/' + path.name + '/' + '/'.join(filename.relative_to(path).parts)
205
- files_dict[relative_path] = file.read()
206
- file.close()
207
-
208
- # Add file data
209
- else:
210
- file = open(path, 'rb')
211
- path_short = '/' + path.name
212
-
213
- files_dict[path_short] = file.read()
214
- file.close()
215
-
216
- elif isinstance(files, dict):
217
- files_dict.update(files)
218
- else:
219
- raise Exception('The given files input must be list or dict or None')
220
-
221
272
  module_input_serialized: bytes = ModuleInput().serialize(
222
273
  stdin=stdin,
223
274
  arguments=args,
@@ -228,7 +279,7 @@ Example: "app.cli('--help')"
228
279
  def _run_locally(self, module_input_serialized: bytes) -> Result:
229
280
  job_dict = BiolibJobApi.create(
230
281
  app_version_id=self._app_version['public_id'],
231
- app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
282
+ app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
232
283
  )
233
284
  job = Result(job_dict)
234
285
 
@@ -253,7 +304,7 @@ Example: "app.cli('--help')"
253
304
  continue
254
305
 
255
306
  if isinstance(value, dict):
256
- value = io.StringIO(json.dumps(value))
307
+ value = JsonStringIO(json.dumps(value))
257
308
  elif isinstance(value, (int, float)): # Cast numeric values to strings
258
309
  value = str(value)
259
310
 
@@ -78,6 +78,7 @@ class _Module(TypedDict):
78
78
  large_file_systems: List[LargeFileSystemMapping]
79
79
  name: str
80
80
  output_files_mappings: List[FilesMapping]
81
+ ports: List[int]
81
82
  source_files_mappings: List[FilesMapping]
82
83
  working_directory: str
83
84