pybiolib 1.1.1974__py3-none-any.whl → 1.1.1984__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/_internal/fuse_mount/experiment_fuse_mount.py +65 -60
- biolib/app/app.py +32 -71
- biolib/biolib_binary_format/remote_endpoints.py +12 -10
- biolib/cli/auth.py +1 -1
- biolib/jobs/job.py +67 -1
- biolib/jobs/job_result.py +4 -3
- {pybiolib-1.1.1974.dist-info → pybiolib-1.1.1984.dist-info}/METADATA +1 -1
- {pybiolib-1.1.1974.dist-info → pybiolib-1.1.1984.dist-info}/RECORD +11 -11
- {pybiolib-1.1.1974.dist-info → pybiolib-1.1.1984.dist-info}/LICENSE +0 -0
- {pybiolib-1.1.1974.dist-info → pybiolib-1.1.1984.dist-info}/WHEEL +0 -0
- {pybiolib-1.1.1974.dist-info → pybiolib-1.1.1984.dist-info}/entry_points.txt +0 -0
@@ -3,12 +3,11 @@ import os
|
|
3
3
|
import stat
|
4
4
|
from datetime import datetime, timezone
|
5
5
|
from time import time
|
6
|
-
from typing import Iterable
|
7
6
|
|
8
7
|
from biolib._internal.libs.fusepy import FUSE, FuseOSError, Operations
|
9
8
|
from biolib.biolib_errors import BioLibError
|
10
9
|
from biolib.jobs import Job
|
11
|
-
from biolib.typing_utils import Dict, Optional, Tuple, TypedDict
|
10
|
+
from biolib.typing_utils import Dict, List, Optional, Tuple, TypedDict
|
12
11
|
|
13
12
|
|
14
13
|
class _AttributeDict(TypedDict):
|
@@ -27,7 +26,6 @@ class ExperimentFuseMount(Operations):
|
|
27
26
|
self._experiment = experiment
|
28
27
|
self._job_names_map: Optional[Dict[str, Job]] = None
|
29
28
|
self._jobs_last_fetched_at: float = 0.0
|
30
|
-
self._root_path: str = '/'
|
31
29
|
self._mounted_at_epoch_seconds: int = int(time())
|
32
30
|
|
33
31
|
@staticmethod
|
@@ -41,80 +39,77 @@ class ExperimentFuseMount(Operations):
|
|
41
39
|
)
|
42
40
|
|
43
41
|
def getattr(self, path: str, fh=None) -> _AttributeDict:
|
44
|
-
|
45
|
-
|
46
|
-
# return folder dir
|
47
|
-
return self._get_folder_attr(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
|
48
|
-
|
49
|
-
job_name, job_path = self._parse_path(path)
|
50
|
-
job = self._get_job_names_map().get(job_name)
|
51
|
-
if not job:
|
52
|
-
# job not found
|
53
|
-
raise FuseOSError(errno.ENOENT)
|
42
|
+
if path == '/':
|
43
|
+
return self._get_directory_attributes(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
|
54
44
|
|
45
|
+
job, path_in_job = self._parse_path(path)
|
55
46
|
job_finished_at_epoch_seconds: int = int(
|
56
47
|
datetime.fromisoformat(job.to_dict()['finished_at'].rstrip('Z')).replace(tzinfo=timezone.utc).timestamp()
|
57
48
|
)
|
58
|
-
|
59
|
-
|
60
|
-
return self.
|
49
|
+
|
50
|
+
if path_in_job == '/':
|
51
|
+
return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
|
61
52
|
|
62
53
|
try:
|
63
|
-
file = job.get_output_file(
|
54
|
+
file = job.get_output_file(path_in_job)
|
55
|
+
return self._get_file_attributes(
|
56
|
+
timestamp_epoch_seconds=job_finished_at_epoch_seconds,
|
57
|
+
size_in_bytes=file.length,
|
58
|
+
)
|
64
59
|
except BioLibError:
|
65
60
|
# file not found
|
66
|
-
|
61
|
+
pass
|
67
62
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
st_mtime=job_finished_at_epoch_seconds,
|
74
|
-
st_nlink=1,
|
75
|
-
st_size=file.length,
|
76
|
-
st_uid=os.getuid(),
|
77
|
-
)
|
63
|
+
file_paths_in_job = [file.path for file in job.list_output_files()]
|
64
|
+
|
65
|
+
for file_path_in_job in file_paths_in_job:
|
66
|
+
if file_path_in_job.startswith(path_in_job):
|
67
|
+
return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
|
78
68
|
|
79
|
-
|
69
|
+
raise FuseOSError(errno.ENOENT) from None # No such file or directory
|
70
|
+
|
71
|
+
def readdir(self, path: str, fh: int) -> List[str]:
|
80
72
|
directory_entries = ['.', '..']
|
73
|
+
|
81
74
|
if path == '/':
|
82
|
-
|
83
|
-
directory_entries.append(name)
|
75
|
+
directory_entries.extend(self._get_job_names_map(refresh_jobs=True).keys())
|
84
76
|
else:
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
77
|
+
job, path_in_job = self._parse_path(path)
|
78
|
+
dir_path_in_job = '/' if path_in_job == '/' else path_in_job + '/'
|
79
|
+
depth = dir_path_in_job.count('/')
|
80
|
+
directory_entries.extend(
|
81
|
+
set(
|
82
|
+
[
|
83
|
+
file.path.split('/')[depth]
|
84
|
+
for file in job.list_output_files()
|
85
|
+
if file.path.startswith(dir_path_in_job)
|
86
|
+
]
|
87
|
+
)
|
89
88
|
)
|
90
89
|
|
91
|
-
|
92
|
-
directory_entries.append(key)
|
93
|
-
|
94
|
-
yield from directory_entries
|
90
|
+
return directory_entries
|
95
91
|
|
96
92
|
def open(self, path: str, flags: int) -> int:
|
97
|
-
|
98
|
-
job = self._get_job_names_map()[job_name]
|
93
|
+
job, path_in_job = self._parse_path(path)
|
99
94
|
try:
|
100
|
-
job.get_output_file(
|
101
|
-
return 0 # return dummy file handle
|
95
|
+
job.get_output_file(path_in_job)
|
102
96
|
except BioLibError:
|
103
97
|
# file not found
|
104
98
|
raise FuseOSError(errno.ENOENT) from None
|
105
99
|
|
100
|
+
return 0 # return dummy file handle
|
101
|
+
|
106
102
|
def read(self, path: str, size: int, offset: int, fh: int) -> bytes:
|
107
|
-
|
108
|
-
job = self._get_job_names_map()[job_name]
|
103
|
+
job, path_in_job = self._parse_path(path)
|
109
104
|
try:
|
110
|
-
file = job.get_output_file(
|
105
|
+
file = job.get_output_file(path_in_job)
|
111
106
|
except BioLibError:
|
112
|
-
# file
|
113
|
-
raise FuseOSError(errno.ENOENT) from None
|
107
|
+
raise FuseOSError(errno.ENOENT) from None # No such file or directory
|
114
108
|
|
115
109
|
return file.get_data(start=offset, length=size)
|
116
110
|
|
117
|
-
|
111
|
+
@staticmethod
|
112
|
+
def _get_directory_attributes(timestamp_epoch_seconds: int) -> _AttributeDict:
|
118
113
|
return _AttributeDict(
|
119
114
|
st_atime=timestamp_epoch_seconds,
|
120
115
|
st_ctime=timestamp_epoch_seconds,
|
@@ -126,6 +121,19 @@ class ExperimentFuseMount(Operations):
|
|
126
121
|
st_uid=os.getuid(),
|
127
122
|
)
|
128
123
|
|
124
|
+
@staticmethod
|
125
|
+
def _get_file_attributes(timestamp_epoch_seconds: int, size_in_bytes: int) -> _AttributeDict:
|
126
|
+
return _AttributeDict(
|
127
|
+
st_atime=timestamp_epoch_seconds,
|
128
|
+
st_ctime=timestamp_epoch_seconds,
|
129
|
+
st_gid=os.getgid(),
|
130
|
+
st_mode=stat.S_IFREG | 0o444, # Regular file with read permissions for owner, group, and others.
|
131
|
+
st_mtime=timestamp_epoch_seconds,
|
132
|
+
st_nlink=1,
|
133
|
+
st_size=size_in_bytes,
|
134
|
+
st_uid=os.getuid(),
|
135
|
+
)
|
136
|
+
|
129
137
|
def _get_job_names_map(self, refresh_jobs=False) -> Dict[str, Job]:
|
130
138
|
current_time = time()
|
131
139
|
if not self._job_names_map or (current_time - self._jobs_last_fetched_at > 1 and refresh_jobs):
|
@@ -134,18 +142,15 @@ class ExperimentFuseMount(Operations):
|
|
134
142
|
|
135
143
|
return self._job_names_map
|
136
144
|
|
137
|
-
def
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
145
|
+
def _parse_path(self, path: str) -> Tuple[Job, str]:
|
146
|
+
path_splitted = path.split('/')
|
147
|
+
job_name = path_splitted[1]
|
148
|
+
path_in_job = '/' + '/'.join(path_splitted[2:])
|
149
|
+
job = self._get_job_names_map().get(job_name)
|
150
|
+
if not job:
|
151
|
+
raise FuseOSError(errno.ENOENT) # No such file or directory
|
142
152
|
|
143
|
-
|
144
|
-
full_path = self._full_path(path)
|
145
|
-
full_path_splitted = full_path.split('/')
|
146
|
-
job_name = full_path_splitted[1]
|
147
|
-
job_path = '/'.join(full_path_splitted[2:])
|
148
|
-
return job_name, job_path
|
153
|
+
return job, path_in_job
|
149
154
|
|
150
155
|
# ----------------------------------- File system methods not implemented below -----------------------------------
|
151
156
|
|
biolib/app/app.py
CHANGED
@@ -1,29 +1,26 @@
|
|
1
|
-
import os
|
2
1
|
import io
|
3
|
-
import random
|
4
2
|
import json
|
3
|
+
import os
|
4
|
+
import random
|
5
5
|
import string
|
6
|
-
|
7
6
|
from pathlib import Path
|
7
|
+
|
8
8
|
from biolib import utils
|
9
|
-
from biolib.
|
10
|
-
from biolib.compute_node.job_worker.job_worker import JobWorker
|
11
|
-
from biolib.experiments.experiment import Experiment
|
12
|
-
from biolib.jobs import Job
|
13
|
-
from biolib.typing_utils import Optional, cast
|
14
|
-
from biolib.biolib_api_client import CreatedJobDict, JobState
|
15
|
-
from biolib.jobs.types import JobDict
|
9
|
+
from biolib.biolib_api_client import JobState
|
16
10
|
from biolib.biolib_api_client.app_types import App, AppVersion
|
17
|
-
from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
|
18
11
|
from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
|
12
|
+
from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
|
19
13
|
from biolib.biolib_binary_format import ModuleInput
|
20
14
|
from biolib.biolib_errors import BioLibError
|
21
15
|
from biolib.biolib_logging import logger
|
16
|
+
from biolib.compute_node.job_worker.job_worker import JobWorker
|
17
|
+
from biolib.experiments.experiment import Experiment
|
18
|
+
from biolib.jobs import Job
|
19
|
+
from biolib.typing_utils import Optional
|
22
20
|
from biolib.utils.app_uri import parse_app_uri
|
23
21
|
|
24
22
|
|
25
23
|
class BioLibApp:
|
26
|
-
|
27
24
|
def __init__(self, uri: str):
|
28
25
|
app_response = BiolibAppApi.get_by_uri(uri)
|
29
26
|
self._app: App = app_response['app']
|
@@ -48,17 +45,17 @@ class BioLibApp:
|
|
48
45
|
return self._app_version
|
49
46
|
|
50
47
|
def cli(
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
48
|
+
self,
|
49
|
+
args=None,
|
50
|
+
stdin=None,
|
51
|
+
files=None,
|
52
|
+
override_command=False,
|
53
|
+
machine='',
|
54
|
+
blocking: bool = True,
|
55
|
+
experiment_id: Optional[str] = None,
|
56
|
+
result_prefix: Optional[str] = None,
|
57
|
+
timeout: Optional[int] = None,
|
58
|
+
notify: bool = False,
|
62
59
|
) -> Job:
|
63
60
|
if not experiment_id:
|
64
61
|
experiment = Experiment.get_experiment_in_context()
|
@@ -78,7 +75,9 @@ class BioLibApp:
|
|
78
75
|
|
79
76
|
return self._run_locally(module_input_serialized)
|
80
77
|
|
81
|
-
job =
|
78
|
+
job = Job._start_job_in_cloud( # pylint: disable=protected-access
|
79
|
+
app_uri=self._app_uri,
|
80
|
+
app_version_uuid=self._app_version['public_id'],
|
82
81
|
experiment_id=experiment_id,
|
83
82
|
machine=machine,
|
84
83
|
module_input_serialized=module_input_serialized,
|
@@ -93,8 +92,8 @@ class BioLibApp:
|
|
93
92
|
utils.STREAM_STDOUT = True
|
94
93
|
|
95
94
|
enable_print = bool(
|
96
|
-
utils.STREAM_STDOUT
|
97
|
-
(self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
|
95
|
+
utils.STREAM_STDOUT
|
96
|
+
and (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
|
98
97
|
)
|
99
98
|
job._stream_logs(enable_print=enable_print) # pylint: disable=protected-access
|
100
99
|
|
@@ -108,11 +107,11 @@ class BioLibApp:
|
|
108
107
|
self.cli()
|
109
108
|
|
110
109
|
else:
|
111
|
-
raise BioLibError(
|
110
|
+
raise BioLibError("""
|
112
111
|
Calling an app directly with app() is currently being reworked.
|
113
112
|
To use the previous functionality, please call app.cli() instead.
|
114
113
|
Example: "app.cli('--help')"
|
115
|
-
|
114
|
+
""")
|
116
115
|
|
117
116
|
@staticmethod
|
118
117
|
def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
|
@@ -142,9 +141,9 @@ Example: "app.cli('--help')"
|
|
142
141
|
args[idx] = Path(arg).name
|
143
142
|
|
144
143
|
# support --myarg=file.txt
|
145
|
-
elif os.path.isfile(arg.split(
|
146
|
-
files.append(arg.split(
|
147
|
-
args[idx] = arg.split(
|
144
|
+
elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
|
145
|
+
files.append(arg.split('=')[-1])
|
146
|
+
args[idx] = arg.split('=')[0] + '=' + Path(arg.split('=')[-1]).name
|
148
147
|
else:
|
149
148
|
pass # a normal string arg was given
|
150
149
|
else:
|
@@ -154,7 +153,7 @@ Example: "app.cli('--help')"
|
|
154
153
|
elif isinstance(arg, io.BytesIO):
|
155
154
|
file_data = arg.getvalue()
|
156
155
|
else:
|
157
|
-
raise Exception(f
|
156
|
+
raise Exception(f'Unexpected type of argument: {arg}')
|
158
157
|
files_dict[f'/{tmp_filename}'] = file_data
|
159
158
|
args[idx] = tmp_filename
|
160
159
|
|
@@ -192,48 +191,10 @@ Example: "app.cli('--help')"
|
|
192
191
|
)
|
193
192
|
return module_input_serialized
|
194
193
|
|
195
|
-
def _start_in_cloud(
|
196
|
-
self,
|
197
|
-
module_input_serialized: bytes,
|
198
|
-
override_command: bool = False,
|
199
|
-
machine: Optional[str] = None,
|
200
|
-
experiment_id: Optional[str] = None,
|
201
|
-
result_prefix: Optional[str] = None,
|
202
|
-
timeout: Optional[int] = None,
|
203
|
-
notify: bool = False,
|
204
|
-
) -> Job:
|
205
|
-
if len(module_input_serialized) < 500_000:
|
206
|
-
_job_dict = BiolibJobApi.create_job_with_data(
|
207
|
-
app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
|
208
|
-
app_version_uuid=self._app_version['public_id'],
|
209
|
-
arguments_override_command=override_command,
|
210
|
-
experiment_uuid=experiment_id,
|
211
|
-
module_input_serialized=module_input_serialized,
|
212
|
-
notify=notify,
|
213
|
-
requested_machine=machine,
|
214
|
-
requested_timeout_seconds=timeout,
|
215
|
-
result_name_prefix=result_prefix,
|
216
|
-
)
|
217
|
-
return Job(cast(JobDict, _job_dict))
|
218
|
-
|
219
|
-
job_dict: CreatedJobDict = BiolibJobApi.create(
|
220
|
-
app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
|
221
|
-
app_version_id=self._app_version['public_id'],
|
222
|
-
experiment_uuid=experiment_id,
|
223
|
-
machine=machine,
|
224
|
-
notify=notify,
|
225
|
-
override_command=override_command,
|
226
|
-
timeout=timeout,
|
227
|
-
)
|
228
|
-
JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
|
229
|
-
cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
|
230
|
-
logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
|
231
|
-
return Job(cast(JobDict, job_dict))
|
232
|
-
|
233
194
|
def _run_locally(self, module_input_serialized: bytes) -> Job:
|
234
195
|
job_dict = BiolibJobApi.create(
|
235
196
|
app_version_id=self._app_version['public_id'],
|
236
|
-
app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix']
|
197
|
+
app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
|
237
198
|
)
|
238
199
|
job = Job(job_dict)
|
239
200
|
|
@@ -1,25 +1,27 @@
|
|
1
1
|
from datetime import datetime, timedelta
|
2
|
-
# from urllib.parse import urlparse, parse_qs
|
3
|
-
|
4
|
-
from biolib.biolib_logging import logger
|
5
2
|
|
6
3
|
from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
|
7
4
|
from biolib.biolib_binary_format.utils import RemoteEndpoint
|
8
5
|
|
6
|
+
# from urllib.parse import urlparse, parse_qs
|
7
|
+
from biolib.biolib_logging import logger
|
8
|
+
from biolib.typing_utils import Literal
|
9
|
+
|
9
10
|
|
10
|
-
class
|
11
|
-
def __init__(self,
|
12
|
-
self._job_id = job_id
|
13
|
-
self._job_auth_token = job_auth_token
|
11
|
+
class RemoteJobStorageEndpoint(RemoteEndpoint):
|
12
|
+
def __init__(self, job_uuid: str, job_auth_token: str, storage_type: Literal['input', 'output']):
|
14
13
|
self._expires_at = None
|
14
|
+
self._job_auth_token = job_auth_token
|
15
|
+
self._job_uuid = job_uuid
|
15
16
|
self._presigned_url = None
|
17
|
+
self._storage_type: Literal['input', 'output'] = storage_type
|
16
18
|
|
17
19
|
def get_remote_url(self):
|
18
20
|
if not self._presigned_url or datetime.utcnow() > self._expires_at:
|
19
21
|
self._presigned_url = BiolibJobApi.get_job_storage_download_url(
|
20
22
|
job_auth_token=self._job_auth_token,
|
21
|
-
job_uuid=self.
|
22
|
-
storage_type='results'
|
23
|
+
job_uuid=self._job_uuid,
|
24
|
+
storage_type='results' if self._storage_type == 'output' else 'input',
|
23
25
|
)
|
24
26
|
self._expires_at = datetime.utcnow() + timedelta(minutes=8)
|
25
27
|
# TODO: Use expires at from url
|
@@ -27,6 +29,6 @@ class RemoteJobStorageResultEndpoint(RemoteEndpoint):
|
|
27
29
|
# query_params = parse_qs(parsed_url.query)
|
28
30
|
# time_at_generation = datetime.datetime.strptime(query_params['X-Amz-Date'][0], '%Y%m%dT%H%M%SZ')
|
29
31
|
# self._expires_at = time_at_generation + timedelta(seconds=int(query_params['X-Amz-Expires'][0]))
|
30
|
-
logger.debug(f'Job "{self.
|
32
|
+
logger.debug(f'Job "{self._job_uuid}" fetched presigned URL with expiry at {self._expires_at.isoformat()}')
|
31
33
|
|
32
34
|
return self._presigned_url
|
biolib/cli/auth.py
CHANGED
@@ -52,7 +52,7 @@ def whoami() -> None:
|
|
52
52
|
email = user_dict['email']
|
53
53
|
intrinsic_account = [account for account in user_dict['accounts'] if account['role'] == 'intrinsic'][0]
|
54
54
|
display_name = intrinsic_account['display_name']
|
55
|
-
print(f'Name: {display_name}\nEmail: {email}')
|
55
|
+
print(f'Name: {display_name}\nEmail: {email}\nLogged into: {client.base_url}')
|
56
56
|
else:
|
57
57
|
print('Not logged in', file=sys.stderr)
|
58
58
|
exit(1)
|
biolib/jobs/job.py
CHANGED
@@ -9,18 +9,22 @@ from urllib.parse import urlparse
|
|
9
9
|
from biolib import api, utils
|
10
10
|
from biolib._internal.http_client import HttpClient
|
11
11
|
from biolib._internal.utils import open_browser_window_from_notebook
|
12
|
-
from biolib.biolib_api_client import BiolibApiClient
|
12
|
+
from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
|
13
|
+
from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
|
13
14
|
from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
|
14
15
|
from biolib.biolib_binary_format import LazyLoadedFile, ModuleInput, ModuleInputDict, ModuleOutputV2
|
16
|
+
from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
|
15
17
|
from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
|
16
18
|
from biolib.biolib_errors import BioLibError, CloudJobFinishedError
|
17
19
|
from biolib.biolib_logging import logger, logger_no_user_data
|
20
|
+
from biolib.compute_node.job_worker.job_storage import JobStorage
|
18
21
|
from biolib.compute_node.utils import SystemExceptionCodeMap, SystemExceptionCodes
|
19
22
|
from biolib.jobs.job_result import JobResult
|
20
23
|
from biolib.jobs.types import CloudJobDict, CloudJobStartedDict, JobDict
|
21
24
|
from biolib.tables import BioLibTable
|
22
25
|
from biolib.typing_utils import Dict, List, Optional, cast
|
23
26
|
from biolib.utils import IS_RUNNING_IN_NOTEBOOK
|
27
|
+
from biolib.utils.app_uri import parse_app_uri
|
24
28
|
|
25
29
|
|
26
30
|
class Job:
|
@@ -198,6 +202,28 @@ class Job:
|
|
198
202
|
except Exception as error:
|
199
203
|
logger.error(f'Failed to cancel job {self._uuid} due to: {error}')
|
200
204
|
|
205
|
+
def recompute(self, app_uri: Optional[str] = None, machine: Optional[str] = None, blocking: bool = True) -> 'Job':
|
206
|
+
app_response = BiolibAppApi.get_by_uri(uri=app_uri or self._job_dict['app_uri'])
|
207
|
+
|
208
|
+
job_storage_input = RemoteJobStorageEndpoint(
|
209
|
+
job_auth_token=self._auth_token,
|
210
|
+
job_uuid=self._uuid,
|
211
|
+
storage_type='input',
|
212
|
+
)
|
213
|
+
http_response = HttpClient.request(url=job_storage_input.get_remote_url())
|
214
|
+
module_input_serialized = http_response.content
|
215
|
+
|
216
|
+
job = self._start_job_in_cloud(
|
217
|
+
app_uri=app_response['app_uri'],
|
218
|
+
app_version_uuid=app_response['app_version']['public_id'],
|
219
|
+
module_input_serialized=module_input_serialized,
|
220
|
+
machine=machine,
|
221
|
+
)
|
222
|
+
if blocking:
|
223
|
+
job.stream_logs()
|
224
|
+
|
225
|
+
return job
|
226
|
+
|
201
227
|
def _get_cloud_job(self) -> CloudJobDict:
|
202
228
|
self._refetch_job_dict(force_refetch=True)
|
203
229
|
if self._job_dict['cloud_job'] is None:
|
@@ -377,3 +403,43 @@ class Job:
|
|
377
403
|
|
378
404
|
self._job_dict = self._get_job_dict(self._uuid, self._auth_token)
|
379
405
|
self._job_dict_last_fetched_at = datetime.utcnow()
|
406
|
+
|
407
|
+
@staticmethod
|
408
|
+
def _start_job_in_cloud(
|
409
|
+
app_uri: str,
|
410
|
+
app_version_uuid: str,
|
411
|
+
module_input_serialized: bytes,
|
412
|
+
override_command: bool = False,
|
413
|
+
machine: Optional[str] = None,
|
414
|
+
experiment_id: Optional[str] = None,
|
415
|
+
result_prefix: Optional[str] = None,
|
416
|
+
timeout: Optional[int] = None,
|
417
|
+
notify: bool = False,
|
418
|
+
) -> 'Job':
|
419
|
+
if len(module_input_serialized) < 500_000:
|
420
|
+
_job_dict = BiolibJobApi.create_job_with_data(
|
421
|
+
app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
|
422
|
+
app_version_uuid=app_version_uuid,
|
423
|
+
arguments_override_command=override_command,
|
424
|
+
experiment_uuid=experiment_id,
|
425
|
+
module_input_serialized=module_input_serialized,
|
426
|
+
notify=notify,
|
427
|
+
requested_machine=machine,
|
428
|
+
requested_timeout_seconds=timeout,
|
429
|
+
result_name_prefix=result_prefix,
|
430
|
+
)
|
431
|
+
return Job(cast(JobDict, _job_dict))
|
432
|
+
|
433
|
+
job_dict: CreatedJobDict = BiolibJobApi.create(
|
434
|
+
app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
|
435
|
+
app_version_id=app_version_uuid,
|
436
|
+
experiment_uuid=experiment_id,
|
437
|
+
machine=machine,
|
438
|
+
notify=notify,
|
439
|
+
override_command=override_command,
|
440
|
+
timeout=timeout,
|
441
|
+
)
|
442
|
+
JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
|
443
|
+
cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
|
444
|
+
logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
|
445
|
+
return Job(cast(JobDict, job_dict))
|
biolib/jobs/job_result.py
CHANGED
@@ -3,7 +3,7 @@ from fnmatch import fnmatch
|
|
3
3
|
from pathlib import Path
|
4
4
|
|
5
5
|
from biolib.biolib_binary_format import ModuleOutputV2
|
6
|
-
from biolib.biolib_binary_format.remote_endpoints import
|
6
|
+
from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
|
7
7
|
from biolib.biolib_binary_format.remote_stream_seeker import StreamSeeker
|
8
8
|
from biolib.biolib_binary_format.utils import LazyLoadedFile, RemoteIndexableBuffer
|
9
9
|
from biolib.biolib_errors import BioLibError
|
@@ -109,9 +109,10 @@ class JobResult:
|
|
109
109
|
|
110
110
|
def _get_module_output(self) -> ModuleOutputV2:
|
111
111
|
if self._module_output is None:
|
112
|
-
remote_job_storage_endpoint =
|
113
|
-
job_id=self._job_uuid,
|
112
|
+
remote_job_storage_endpoint = RemoteJobStorageEndpoint(
|
114
113
|
job_auth_token=self._job_auth_token,
|
114
|
+
job_uuid=self._job_uuid,
|
115
|
+
storage_type='output',
|
115
116
|
)
|
116
117
|
buffer = RemoteIndexableBuffer(endpoint=remote_job_storage_endpoint)
|
117
118
|
self._module_output = ModuleOutputV2(buffer)
|
@@ -6,7 +6,7 @@ biolib/_internal/data_record/__init__.py,sha256=1Bk303i3rFet9veS56fIsrBYtT5X3n9v
|
|
6
6
|
biolib/_internal/data_record/data_record.py,sha256=ctijrrZ-LfUxtwzS8PEVa1VBuBLVWEhmo2yHcEDkC0A,7178
|
7
7
|
biolib/_internal/data_record/remote_storage_endpoint.py,sha256=LPq8Lr5FhKF9_o5K-bUdT7TeLe5XFUD0AAeTkNEVZug,1133
|
8
8
|
biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
|
9
|
-
biolib/_internal/fuse_mount/experiment_fuse_mount.py,sha256=
|
9
|
+
biolib/_internal/fuse_mount/experiment_fuse_mount.py,sha256=DcR2NyUyZYgpsaKM1WBJZChAaD2vvt_vvZCXj-P2UTs,6878
|
10
10
|
biolib/_internal/http_client.py,sha256=DdooXei93JKGYGV4aQmzue_oFzvHkozg2UCxgk9dfDM,5081
|
11
11
|
biolib/_internal/libs/__init__.py,sha256=Jdf4tNPqe_oIIf6zYml6TiqhL_02Vyqwge6IELrAFhw,98
|
12
12
|
biolib/_internal/libs/fusepy/__init__.py,sha256=AWDzNFS-XV_5yKb0Qx7kggIhPzq1nj_BZS5y2Nso08k,41944
|
@@ -16,7 +16,7 @@ biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd
|
|
16
16
|
biolib/api/__init__.py,sha256=iIO8ZRdn7YDhY5cR47-Wo1YsNOK8H6RN6jn8yor5WJI,137
|
17
17
|
biolib/api/client.py,sha256=MtDkH2Amr2Fko-bCR5DdookJu0yZ1q-6K_PPg4KK_Ek,2941
|
18
18
|
biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
|
19
|
-
biolib/app/app.py,sha256=
|
19
|
+
biolib/app/app.py,sha256=8AvPYL1W2wxQ7t7BB2KeVU2WPrm3UL6vVuHPGs8g9L0,8388
|
20
20
|
biolib/app/search_apps.py,sha256=K4a41f5XIWth2BWI7OffASgIsD0ko8elCax8YL2igaY,1470
|
21
21
|
biolib/biolib_api_client/__init__.py,sha256=E5EMa19wJoblwSdQPYrxc_BtIeRsAuO0L_jQweWw-Yk,182
|
22
22
|
biolib/biolib_api_client/api_client.py,sha256=J03jRVvod1bgwwAZ3BZVKlUSJi43-ev2DUB0j63GZpc,7189
|
@@ -33,7 +33,7 @@ biolib/biolib_binary_format/base_bbf_package.py,sha256=vxRV4iKy0dXeDOlFWnMFI0hGn
|
|
33
33
|
biolib/biolib_binary_format/file_in_container.py,sha256=j1eEPRxf_ew8I6G8sDiiZZxn4Wx1ppagfM9K8zTDG4U,3591
|
34
34
|
biolib/biolib_binary_format/module_input.py,sha256=led2QhHeec_ymBPw5uEn3_3vJKI-1T8zrFQGqwEWLMY,2788
|
35
35
|
biolib/biolib_binary_format/module_output_v2.py,sha256=J5ZO5gCSeudpE12EVDrjYrNTS2DwgszY-SVXT7Qjuyg,5913
|
36
|
-
biolib/biolib_binary_format/remote_endpoints.py,sha256=
|
36
|
+
biolib/biolib_binary_format/remote_endpoints.py,sha256=V48mwOj3eAQAKp-8DjtWUdEKUyC0WKc1pEiKTmtjrJY,1651
|
37
37
|
biolib/biolib_binary_format/remote_stream_seeker.py,sha256=uyi6kJBU1C1DWfiuR0kRUQIY7nalG7ocgwgngd3Ul4U,1999
|
38
38
|
biolib/biolib_binary_format/saved_job.py,sha256=nFHVFRNTNcAFGODLSiBntCtMk55QKwreUq6qLX80dI4,1125
|
39
39
|
biolib/biolib_binary_format/stdout_and_stderr.py,sha256=WfUUJFFCBrtfXjuWIaRPiWCpuBLxfko68oxoTKhrwx8,1023
|
@@ -45,7 +45,7 @@ biolib/biolib_download_container.py,sha256=8TmBV8iv3bCvkNlHa1SSsc4zl0wX_eaxhfnW5
|
|
45
45
|
biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
|
46
46
|
biolib/biolib_logging.py,sha256=J3E5H_LL5k6ZUim2C8gqN7E6lCBZMTpO4tnMpOPwG9U,2854
|
47
47
|
biolib/cli/__init__.py,sha256=0v3c_J-U0k46c5ZWeQjLG_kTaKDJm81LBxQpDO2B_aI,1286
|
48
|
-
biolib/cli/auth.py,sha256=
|
48
|
+
biolib/cli/auth.py,sha256=rpWGmXs6Fz6CGrO9K8ibPRszOdXG78Vig_boKaVCD9A,2082
|
49
49
|
biolib/cli/data_record.py,sha256=piN3QUbRAkMi4wpayghN4unFfuiNE5VCjI1gag4d8qg,1725
|
50
50
|
biolib/cli/download_container.py,sha256=HIZVHOPmslGE5M2Dsp9r2cCkAEJx__vcsDz5Wt5LRos,483
|
51
51
|
biolib/cli/init.py,sha256=wQOfii_au-d30Hp7DdH-WVw-WVraKvA_zY4za1w7DE8,821
|
@@ -89,8 +89,8 @@ biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
89
89
|
biolib/experiments/experiment.py,sha256=kUQsH9AGAckPKT_nzaRuTh8Mb2pVUpxnuX9IstRTOEo,6351
|
90
90
|
biolib/experiments/types.py,sha256=n9GxdFA7cLMfHvLLqLmZzX31ELeSSkMXFoEEdFsdWGY,171
|
91
91
|
biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
|
92
|
-
biolib/jobs/job.py,sha256
|
93
|
-
biolib/jobs/job_result.py,sha256=
|
92
|
+
biolib/jobs/job.py,sha256=aWKnf_2pYdr76gh3hxPiVs2iuXlpwZkKPTK81Pz4G2U,19072
|
93
|
+
biolib/jobs/job_result.py,sha256=UGxE9MNLtwJiWmhw2UNjOQW7EZi7B-e2lL0PXYvsXeA,4925
|
94
94
|
biolib/jobs/types.py,sha256=qhadtH2KDC2WUOOqPiwke0YgtQY4FtuB71Stekq1k48,970
|
95
95
|
biolib/lfs/__init__.py,sha256=Qv8vdYeK43JecT4SsE93ZYE2VmNiZENdNpW8P9-omxs,115
|
96
96
|
biolib/lfs/cache.py,sha256=pQS2np21rdJ6I3DpoOutnzPHpLOZgUIS8TMltUJk_k4,2226
|
@@ -109,8 +109,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
|
|
109
109
|
biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
|
110
110
|
biolib/utils/seq_util.py,sha256=jC5WhH63FTD7SLFJbxQGA2hOt9NTwq9zHl_BEec1Z0c,4907
|
111
111
|
biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
|
112
|
-
pybiolib-1.1.
|
113
|
-
pybiolib-1.1.
|
114
|
-
pybiolib-1.1.
|
115
|
-
pybiolib-1.1.
|
116
|
-
pybiolib-1.1.
|
112
|
+
pybiolib-1.1.1984.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
|
113
|
+
pybiolib-1.1.1984.dist-info/METADATA,sha256=PO0Sif5QkYXg1P20p7306bX-CXn6-Mpee7epEsP-1Fo,1508
|
114
|
+
pybiolib-1.1.1984.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
115
|
+
pybiolib-1.1.1984.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
|
116
|
+
pybiolib-1.1.1984.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|