pybiolib 1.1.1747__py3-none-any.whl → 1.1.2193__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/__init__.py +18 -5
- biolib/_data_record/data_record.py +278 -0
- biolib/_internal/data_record/__init__.py +1 -0
- biolib/_internal/data_record/data_record.py +97 -0
- biolib/_internal/data_record/remote_storage_endpoint.py +38 -0
- biolib/_internal/file_utils.py +77 -0
- biolib/_internal/fuse_mount/__init__.py +1 -0
- biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
- biolib/_internal/http_client.py +42 -23
- biolib/_internal/lfs/__init__.py +1 -0
- biolib/_internal/libs/__init__.py +1 -0
- biolib/_internal/libs/fusepy/__init__.py +1257 -0
- biolib/_internal/push_application.py +22 -37
- biolib/_internal/runtime.py +19 -0
- biolib/_internal/types/__init__.py +4 -0
- biolib/_internal/types/app.py +9 -0
- biolib/_internal/types/data_record.py +40 -0
- biolib/_internal/types/experiment.py +10 -0
- biolib/_internal/types/resource.py +14 -0
- biolib/_internal/types/typing.py +7 -0
- biolib/_internal/utils/__init__.py +18 -0
- biolib/_runtime/runtime.py +80 -0
- biolib/api/__init__.py +1 -0
- biolib/api/client.py +39 -17
- biolib/app/app.py +40 -72
- biolib/app/search_apps.py +8 -12
- biolib/biolib_api_client/api_client.py +22 -10
- biolib/biolib_api_client/app_types.py +2 -1
- biolib/biolib_api_client/biolib_app_api.py +1 -1
- biolib/biolib_api_client/biolib_job_api.py +6 -0
- biolib/biolib_api_client/job_types.py +4 -4
- biolib/biolib_api_client/lfs_types.py +8 -2
- biolib/biolib_binary_format/remote_endpoints.py +12 -10
- biolib/biolib_binary_format/utils.py +41 -4
- biolib/cli/__init__.py +6 -2
- biolib/cli/auth.py +58 -0
- biolib/cli/data_record.py +80 -0
- biolib/cli/download_container.py +3 -1
- biolib/cli/init.py +1 -0
- biolib/cli/lfs.py +45 -11
- biolib/cli/push.py +1 -1
- biolib/cli/run.py +3 -2
- biolib/cli/start.py +1 -0
- biolib/compute_node/cloud_utils/cloud_utils.py +15 -18
- biolib/compute_node/job_worker/cache_state.py +1 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +134 -114
- biolib/compute_node/job_worker/job_storage.py +3 -4
- biolib/compute_node/job_worker/job_worker.py +31 -15
- biolib/compute_node/remote_host_proxy.py +75 -70
- biolib/compute_node/webserver/webserver_types.py +0 -1
- biolib/experiments/experiment.py +75 -44
- biolib/jobs/job.py +125 -47
- biolib/jobs/job_result.py +46 -21
- biolib/jobs/types.py +1 -1
- biolib/runtime/__init__.py +14 -1
- biolib/sdk/__init__.py +29 -5
- biolib/typing_utils.py +2 -7
- biolib/user/sign_in.py +10 -14
- biolib/utils/__init__.py +1 -1
- biolib/utils/app_uri.py +11 -4
- biolib/utils/cache_state.py +2 -2
- biolib/utils/seq_util.py +38 -30
- {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/METADATA +1 -1
- pybiolib-1.1.2193.dist-info/RECORD +123 -0
- {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/WHEEL +1 -1
- biolib/biolib_api_client/biolib_account_api.py +0 -8
- biolib/biolib_api_client/biolib_large_file_system_api.py +0 -34
- biolib/experiments/types.py +0 -9
- biolib/lfs/__init__.py +0 -6
- biolib/lfs/utils.py +0 -237
- biolib/runtime/results.py +0 -20
- pybiolib-1.1.1747.dist-info/RECORD +0 -108
- /biolib/{lfs → _internal/lfs}/cache.py +0 -0
- {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/LICENSE +0 -0
- {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/entry_points.txt +0 -0
biolib/jobs/job.py
CHANGED
@@ -1,34 +1,42 @@
|
|
1
1
|
import base64
|
2
|
-
from datetime import datetime, timedelta
|
3
2
|
import sys
|
4
3
|
import time
|
5
|
-
from pathlib import Path
|
6
4
|
from collections import OrderedDict
|
5
|
+
from datetime import datetime, timedelta
|
6
|
+
from pathlib import Path
|
7
7
|
from urllib.parse import urlparse
|
8
8
|
|
9
9
|
from biolib import api, utils
|
10
10
|
from biolib._internal.http_client import HttpClient
|
11
|
+
from biolib._internal.utils import open_browser_window_from_notebook
|
12
|
+
from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
|
13
|
+
from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
|
11
14
|
from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
|
12
|
-
from biolib.biolib_binary_format import LazyLoadedFile,
|
15
|
+
from biolib.biolib_binary_format import LazyLoadedFile, ModuleInput, ModuleInputDict, ModuleOutputV2
|
16
|
+
from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
|
13
17
|
from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
|
14
18
|
from biolib.biolib_errors import BioLibError, CloudJobFinishedError
|
15
19
|
from biolib.biolib_logging import logger, logger_no_user_data
|
20
|
+
from biolib.compute_node.job_worker.job_storage import JobStorage
|
16
21
|
from biolib.compute_node.utils import SystemExceptionCodeMap, SystemExceptionCodes
|
17
22
|
from biolib.jobs.job_result import JobResult
|
18
|
-
from biolib.jobs.types import
|
23
|
+
from biolib.jobs.types import CloudJobDict, CloudJobStartedDict, JobDict
|
19
24
|
from biolib.tables import BioLibTable
|
20
|
-
from biolib.typing_utils import
|
25
|
+
from biolib.typing_utils import Dict, List, Optional, cast
|
21
26
|
from biolib.utils import IS_RUNNING_IN_NOTEBOOK
|
27
|
+
from biolib.utils.app_uri import parse_app_uri
|
22
28
|
|
23
29
|
|
24
30
|
class Job:
|
25
31
|
# Columns to print in table when showing Job
|
26
|
-
table_columns_to_row_map = OrderedDict(
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
+
table_columns_to_row_map = OrderedDict(
|
33
|
+
{
|
34
|
+
'ID': {'key': 'uuid', 'params': {'width': 36}},
|
35
|
+
'Application': {'key': 'app_uri', 'params': {}},
|
36
|
+
'Status': {'key': 'state', 'params': {}},
|
37
|
+
'Started At': {'key': 'started_at', 'params': {}},
|
38
|
+
}
|
39
|
+
)
|
32
40
|
|
33
41
|
def __init__(self, job_dict: JobDict):
|
34
42
|
self._uuid: str = job_dict['uuid']
|
@@ -52,26 +60,23 @@ class Job:
|
|
52
60
|
@property
|
53
61
|
def result(self) -> JobResult:
|
54
62
|
if not self._result:
|
55
|
-
|
56
|
-
self._result = JobResult(job_uuid=self._uuid, job_auth_token=self._auth_token)
|
57
|
-
else:
|
58
|
-
raise BioLibError(f"Result is not available for {self._uuid}: status is {self._job_dict['state']}.")
|
63
|
+
self._result = JobResult(job_uuid=self._uuid, job_auth_token=self._auth_token)
|
59
64
|
|
60
65
|
return self._result
|
61
66
|
|
62
67
|
@property
|
63
68
|
def stdout(self) -> bytes:
|
64
|
-
logger.warning(
|
69
|
+
logger.warning('The property .stdout is deprecated, please use .get_stdout()')
|
65
70
|
return self.result.get_stdout()
|
66
71
|
|
67
72
|
@property
|
68
73
|
def stderr(self) -> bytes:
|
69
|
-
logger.warning(
|
74
|
+
logger.warning('The property .stderr is deprecated, please use .get_stderr()')
|
70
75
|
return self.result.get_stderr()
|
71
76
|
|
72
77
|
@property
|
73
78
|
def exitcode(self) -> int:
|
74
|
-
logger.warning(
|
79
|
+
logger.warning('The property .exitcode is deprecated, please use .get_exit_code()')
|
75
80
|
return self.result.get_exit_code()
|
76
81
|
|
77
82
|
def is_finished(self) -> bool:
|
@@ -105,8 +110,8 @@ class Job:
|
|
105
110
|
def load_file_as_numpy(self, *args, **kwargs):
|
106
111
|
try:
|
107
112
|
import numpy # type: ignore # pylint: disable=import-outside-toplevel,import-error
|
108
|
-
except: # pylint: disable=raise-missing-from
|
109
|
-
raise Exception(
|
113
|
+
except ImportError: # pylint: disable=raise-missing-from
|
114
|
+
raise Exception('Failed to import numpy, please make sure it is installed.') from None
|
110
115
|
file_handle = self.result.get_output_file(*args, **kwargs).get_file_handle()
|
111
116
|
return numpy.load(file_handle, allow_pickle=False) # type: ignore
|
112
117
|
|
@@ -172,6 +177,50 @@ class Job:
|
|
172
177
|
time.sleep(2)
|
173
178
|
logger.info(f'Job {self.id} has finished.')
|
174
179
|
|
180
|
+
def open_browser(self) -> None:
|
181
|
+
api_client = BiolibApiClient.get()
|
182
|
+
results_url_to_open = f'{api_client.base_url}/results/{self.id}/?token={self._auth_token}'
|
183
|
+
if IS_RUNNING_IN_NOTEBOOK:
|
184
|
+
print(f'Opening results page at: {results_url_to_open}')
|
185
|
+
print('If your browser does not open automatically, click on the link above.')
|
186
|
+
open_browser_window_from_notebook(results_url_to_open)
|
187
|
+
else:
|
188
|
+
print('Please copy and paste the following link into your browser:')
|
189
|
+
print(results_url_to_open)
|
190
|
+
|
191
|
+
def cancel(self) -> None:
|
192
|
+
try:
|
193
|
+
api.client.patch(
|
194
|
+
path=f'/jobs/{self._uuid}/',
|
195
|
+
headers={'Job-Auth-Token': self._auth_token} if self._auth_token else None,
|
196
|
+
data={'state': 'cancelled'},
|
197
|
+
)
|
198
|
+
logger.info(f'Job {self._uuid} canceled')
|
199
|
+
except Exception as error:
|
200
|
+
logger.error(f'Failed to cancel job {self._uuid} due to: {error}')
|
201
|
+
|
202
|
+
def recompute(self, app_uri: Optional[str] = None, machine: Optional[str] = None, blocking: bool = True) -> 'Job':
|
203
|
+
app_response = BiolibAppApi.get_by_uri(uri=app_uri or self._job_dict['app_uri'])
|
204
|
+
|
205
|
+
job_storage_input = RemoteJobStorageEndpoint(
|
206
|
+
job_auth_token=self._auth_token,
|
207
|
+
job_uuid=self._uuid,
|
208
|
+
storage_type='input',
|
209
|
+
)
|
210
|
+
http_response = HttpClient.request(url=job_storage_input.get_remote_url())
|
211
|
+
module_input_serialized = http_response.content
|
212
|
+
|
213
|
+
job = self._start_job_in_cloud(
|
214
|
+
app_uri=app_response['app_uri'],
|
215
|
+
app_version_uuid=app_response['app_version']['public_id'],
|
216
|
+
module_input_serialized=module_input_serialized,
|
217
|
+
machine=machine,
|
218
|
+
)
|
219
|
+
if blocking:
|
220
|
+
job.stream_logs()
|
221
|
+
|
222
|
+
return job
|
223
|
+
|
175
224
|
def _get_cloud_job(self) -> CloudJobDict:
|
176
225
|
self._refetch_job_dict(force_refetch=True)
|
177
226
|
if self._job_dict['cloud_job'] is None:
|
@@ -190,20 +239,11 @@ class Job:
|
|
190
239
|
@staticmethod
|
191
240
|
def show_jobs(count: int = 25) -> None:
|
192
241
|
job_dicts = Job._get_job_dicts(count)
|
193
|
-
BioLibTable(
|
194
|
-
columns_to_row_map=Job.table_columns_to_row_map,
|
195
|
-
rows=job_dicts,
|
196
|
-
title='Jobs'
|
197
|
-
).print_table()
|
242
|
+
BioLibTable(columns_to_row_map=Job.table_columns_to_row_map, rows=job_dicts, title='Jobs').print_table()
|
198
243
|
|
199
244
|
@staticmethod
|
200
245
|
def _get_job_dicts(count: int) -> List['JobDict']:
|
201
|
-
job_dicts: List['JobDict'] = api.client.get(
|
202
|
-
path='/jobs/',
|
203
|
-
params={
|
204
|
-
'page_size': str(count)
|
205
|
-
}
|
206
|
-
).json()['results']
|
246
|
+
job_dicts: List['JobDict'] = api.client.get(path='/jobs/', params={'page_size': str(count)}).json()['results']
|
207
247
|
return job_dicts
|
208
248
|
|
209
249
|
@staticmethod
|
@@ -235,9 +275,7 @@ class Job:
|
|
235
275
|
def show(self) -> None:
|
236
276
|
self._refetch_job_dict()
|
237
277
|
BioLibTable(
|
238
|
-
columns_to_row_map=Job.table_columns_to_row_map,
|
239
|
-
rows=[self._job_dict],
|
240
|
-
title=f'Job: {self._uuid}'
|
278
|
+
columns_to_row_map=Job.table_columns_to_row_map, rows=[self._job_dict], title=f'Job: {self._uuid}'
|
241
279
|
).print_table()
|
242
280
|
|
243
281
|
def stream_logs(self) -> None:
|
@@ -274,7 +312,7 @@ class Job:
|
|
274
312
|
status_json = self._get_job_status_from_compute_node(compute_node_url)
|
275
313
|
if not status_json:
|
276
314
|
# this can happen if the job is finished but already removed from the compute node
|
277
|
-
logger.warning(
|
315
|
+
logger.warning('WARN: We were unable to retrieve the full log of the job, please try again')
|
278
316
|
break
|
279
317
|
job_is_completed = status_json['is_completed']
|
280
318
|
for status_update in status_json['status_updates']:
|
@@ -305,12 +343,10 @@ class Job:
|
|
305
343
|
|
306
344
|
def _print_full_logs(self, node_url: str) -> None:
|
307
345
|
try:
|
308
|
-
response_json = HttpClient.request(
|
309
|
-
url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full'
|
310
|
-
).json()
|
346
|
+
response_json = HttpClient.request(url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full').json()
|
311
347
|
except Exception as error:
|
312
348
|
logger.error(f'Could not get full streamed logs due to: {error}')
|
313
|
-
raise BioLibError from error
|
349
|
+
raise BioLibError('Could not get full streamed logs') from error
|
314
350
|
|
315
351
|
for status_update in response_json.get('previous_status_updates', []):
|
316
352
|
logger.info(f'Cloud: {status_update["log_message"]}')
|
@@ -318,7 +354,10 @@ class Job:
|
|
318
354
|
self.print_logs_packages(response_json['streamed_logs_packages_b64'])
|
319
355
|
|
320
356
|
def _get_cloud_job_awaiting_started(self) -> CloudJobStartedDict:
|
357
|
+
retry_count = 0
|
321
358
|
while True:
|
359
|
+
retry_count += 1
|
360
|
+
time.sleep(min(10, retry_count))
|
322
361
|
cloud_job = self._get_cloud_job()
|
323
362
|
|
324
363
|
if cloud_job['finished_at']:
|
@@ -331,23 +370,19 @@ class Job:
|
|
331
370
|
return cast(CloudJobStartedDict, cloud_job)
|
332
371
|
|
333
372
|
logger.info('Cloud: The job has been queued. Please wait...')
|
334
|
-
time.sleep(10)
|
335
373
|
|
336
374
|
def _get_job_status_from_compute_node(self, compute_node_url):
|
337
375
|
for _ in range(15):
|
338
376
|
try:
|
339
|
-
return HttpClient.request(
|
340
|
-
url=f'{compute_node_url}/v1/job/{self._uuid}/status/'
|
341
|
-
).json()
|
377
|
+
return HttpClient.request(url=f'{compute_node_url}/v1/job/{self._uuid}/status/').json()
|
342
378
|
except Exception: # pylint: disable=broad-except
|
343
379
|
cloud_job = self._get_cloud_job()
|
344
|
-
logger.debug(
|
380
|
+
logger.debug('Failed to get status from compute node, retrying...')
|
345
381
|
if cloud_job['finished_at']:
|
346
|
-
logger.debug(
|
382
|
+
logger.debug('Job no longer exists on compute node, checking for error...')
|
347
383
|
if cloud_job['error_code'] != SystemExceptionCodes.COMPLETED_SUCCESSFULLY.value:
|
348
384
|
error_message = SystemExceptionCodeMap.get(
|
349
|
-
cloud_job['error_code'],
|
350
|
-
f'Unknown error code {cloud_job["error_code"]}'
|
385
|
+
cloud_job['error_code'], f'Unknown error code {cloud_job["error_code"]}'
|
351
386
|
)
|
352
387
|
raise BioLibError(f'Cloud: {error_message}') from None
|
353
388
|
else:
|
@@ -367,3 +402,46 @@ class Job:
|
|
367
402
|
|
368
403
|
self._job_dict = self._get_job_dict(self._uuid, self._auth_token)
|
369
404
|
self._job_dict_last_fetched_at = datetime.utcnow()
|
405
|
+
|
406
|
+
@staticmethod
|
407
|
+
def _start_job_in_cloud(
|
408
|
+
app_uri: str,
|
409
|
+
app_version_uuid: str,
|
410
|
+
module_input_serialized: bytes,
|
411
|
+
override_command: bool = False,
|
412
|
+
machine: Optional[str] = None,
|
413
|
+
experiment_id: Optional[str] = None,
|
414
|
+
result_prefix: Optional[str] = None,
|
415
|
+
timeout: Optional[int] = None,
|
416
|
+
notify: bool = False,
|
417
|
+
requested_machine_count: Optional[int] = None,
|
418
|
+
) -> 'Job':
|
419
|
+
if len(module_input_serialized) < 500_000:
|
420
|
+
_job_dict = BiolibJobApi.create_job_with_data(
|
421
|
+
app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
|
422
|
+
app_version_uuid=app_version_uuid,
|
423
|
+
arguments_override_command=override_command,
|
424
|
+
experiment_uuid=experiment_id,
|
425
|
+
module_input_serialized=module_input_serialized,
|
426
|
+
notify=notify,
|
427
|
+
requested_machine=machine,
|
428
|
+
requested_timeout_seconds=timeout,
|
429
|
+
result_name_prefix=result_prefix,
|
430
|
+
requested_machine_count=requested_machine_count,
|
431
|
+
)
|
432
|
+
return Job(cast(JobDict, _job_dict))
|
433
|
+
|
434
|
+
job_dict: CreatedJobDict = BiolibJobApi.create(
|
435
|
+
app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
|
436
|
+
app_version_id=app_version_uuid,
|
437
|
+
experiment_uuid=experiment_id,
|
438
|
+
machine=machine,
|
439
|
+
notify=notify,
|
440
|
+
override_command=override_command,
|
441
|
+
timeout=timeout,
|
442
|
+
requested_machine_count=requested_machine_count,
|
443
|
+
)
|
444
|
+
JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
|
445
|
+
cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
|
446
|
+
logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
|
447
|
+
return Job(cast(JobDict, job_dict))
|
biolib/jobs/job_result.py
CHANGED
@@ -1,25 +1,24 @@
|
|
1
|
-
from pathlib import Path
|
2
|
-
from fnmatch import fnmatch
|
3
1
|
import time
|
2
|
+
from fnmatch import fnmatch
|
3
|
+
from pathlib import Path
|
4
4
|
|
5
5
|
from biolib.biolib_binary_format import ModuleOutputV2
|
6
|
+
from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
|
6
7
|
from biolib.biolib_binary_format.remote_stream_seeker import StreamSeeker
|
7
|
-
from biolib.biolib_binary_format.utils import
|
8
|
-
from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageResultEndpoint
|
8
|
+
from biolib.biolib_binary_format.utils import LazyLoadedFile, RemoteIndexableBuffer
|
9
9
|
from biolib.biolib_errors import BioLibError
|
10
10
|
from biolib.biolib_logging import logger
|
11
|
-
from biolib.typing_utils import
|
11
|
+
from biolib.typing_utils import Callable, List, Optional, Union, cast
|
12
12
|
|
13
13
|
PathFilter = Union[str, Callable[[str], bool]]
|
14
14
|
|
15
15
|
|
16
16
|
class JobResult:
|
17
|
-
|
18
17
|
def __init__(
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
18
|
+
self,
|
19
|
+
job_uuid: str,
|
20
|
+
job_auth_token: str,
|
21
|
+
module_output: Optional[ModuleOutputV2] = None,
|
23
22
|
):
|
24
23
|
self._job_uuid: str = job_uuid
|
25
24
|
self._job_auth_token: str = job_auth_token
|
@@ -35,7 +34,12 @@ class JobResult:
|
|
35
34
|
def get_exit_code(self) -> int:
|
36
35
|
return self._get_module_output().get_exit_code()
|
37
36
|
|
38
|
-
def save_files(
|
37
|
+
def save_files(
|
38
|
+
self,
|
39
|
+
output_dir: str,
|
40
|
+
path_filter: Optional[PathFilter] = None,
|
41
|
+
skip_file_if_exists: Optional[bool] = None,
|
42
|
+
) -> None:
|
39
43
|
module_output = self._get_module_output()
|
40
44
|
output_files = module_output.get_files()
|
41
45
|
filtered_output_files = self._get_filtered_files(output_files, path_filter) if path_filter else output_files
|
@@ -61,24 +65,44 @@ class JobResult:
|
|
61
65
|
# Remove leading slash of file_path
|
62
66
|
destination_file_path = Path(output_dir) / Path(file.path.lstrip('/'))
|
63
67
|
if destination_file_path.exists():
|
64
|
-
|
68
|
+
if skip_file_if_exists:
|
69
|
+
print(f'Skipping {destination_file_path} as a file with that name already exists locally.')
|
70
|
+
continue
|
71
|
+
else:
|
72
|
+
destination_file_path.rename(
|
73
|
+
f'{destination_file_path}.biolib-renamed.{time.strftime("%Y%m%d%H%M%S")}'
|
74
|
+
)
|
65
75
|
|
66
76
|
dir_path = destination_file_path.parent
|
67
77
|
if dir_path:
|
68
78
|
dir_path.mkdir(parents=True, exist_ok=True)
|
69
79
|
|
70
|
-
|
71
|
-
|
72
|
-
|
80
|
+
# write content to temporary (partial) file
|
81
|
+
partial_path = destination_file_path.with_suffix(
|
82
|
+
destination_file_path.suffix + f'.{self._job_uuid}.partial_biolib_download'
|
83
|
+
)
|
84
|
+
file_start = file.start
|
85
|
+
data_to_download = file.length
|
86
|
+
if partial_path.exists():
|
87
|
+
data_already_downloaded = partial_path.stat().st_size
|
88
|
+
file_start += data_already_downloaded
|
89
|
+
data_to_download -= data_already_downloaded
|
90
|
+
|
91
|
+
with open(partial_path, mode='ab') as partial_file:
|
92
|
+
for chunk in stream_seeker.seek_and_read(file_start=file_start, file_length=data_to_download):
|
93
|
+
partial_file.write(chunk)
|
94
|
+
|
95
|
+
# rename partial file to actual file name
|
96
|
+
partial_path.rename(destination_file_path)
|
73
97
|
|
74
98
|
def get_output_file(self, filename) -> LazyLoadedFile:
|
75
99
|
files = self._get_module_output().get_files()
|
76
100
|
filtered_files = self._get_filtered_files(files, path_filter=filename)
|
77
101
|
if not filtered_files:
|
78
|
-
raise BioLibError(f
|
102
|
+
raise BioLibError(f'File {filename} not found in results.')
|
79
103
|
|
80
104
|
if len(filtered_files) != 1:
|
81
|
-
raise BioLibError(f
|
105
|
+
raise BioLibError(f'Found multiple results for filename {filename}.')
|
82
106
|
|
83
107
|
return filtered_files[0]
|
84
108
|
|
@@ -100,8 +124,8 @@ class JobResult:
|
|
100
124
|
glob_filter = cast(str, path_filter)
|
101
125
|
|
102
126
|
# since all file paths start with /, make sure filter does too
|
103
|
-
if not glob_filter.startswith(
|
104
|
-
glob_filter =
|
127
|
+
if not glob_filter.startswith('/'):
|
128
|
+
glob_filter = '/' + glob_filter
|
105
129
|
|
106
130
|
def _filter_function(file: LazyLoadedFile) -> bool:
|
107
131
|
return fnmatch(file.path, glob_filter)
|
@@ -110,9 +134,10 @@ class JobResult:
|
|
110
134
|
|
111
135
|
def _get_module_output(self) -> ModuleOutputV2:
|
112
136
|
if self._module_output is None:
|
113
|
-
remote_job_storage_endpoint =
|
114
|
-
job_id=self._job_uuid,
|
137
|
+
remote_job_storage_endpoint = RemoteJobStorageEndpoint(
|
115
138
|
job_auth_token=self._job_auth_token,
|
139
|
+
job_uuid=self._job_uuid,
|
140
|
+
storage_type='output',
|
116
141
|
)
|
117
142
|
buffer = RemoteIndexableBuffer(endpoint=remote_job_storage_endpoint)
|
118
143
|
self._module_output = ModuleOutputV2(buffer)
|
biolib/jobs/types.py
CHANGED
biolib/runtime/__init__.py
CHANGED
@@ -1 +1,14 @@
|
|
1
|
-
|
1
|
+
import warnings
|
2
|
+
|
3
|
+
from biolib._runtime.runtime import Runtime as _Runtime
|
4
|
+
|
5
|
+
|
6
|
+
def set_main_result_prefix(result_prefix: str) -> None:
|
7
|
+
warnings.warn(
|
8
|
+
'The "biolib.runtime.set_main_result_prefix" function is deprecated. '
|
9
|
+
'It will be removed in future releases from mid 2024. '
|
10
|
+
'Please use "from biolib.sdk import Runtime" and then "Runtime.set_main_result_prefix" instead.',
|
11
|
+
DeprecationWarning,
|
12
|
+
stacklevel=2,
|
13
|
+
)
|
14
|
+
_Runtime.set_main_result_prefix(result_prefix)
|
biolib/sdk/__init__.py
CHANGED
@@ -1,33 +1,57 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
# Imports to hide and use as private internal utils
|
4
|
+
from biolib._data_record.data_record import DataRecord as _DataRecord
|
1
5
|
from biolib._internal.push_application import push_application as _push_application
|
2
6
|
from biolib._internal.push_application import set_app_version_as_active as _set_app_version_as_active
|
3
|
-
|
7
|
+
from biolib._runtime.runtime import Runtime as _Runtime
|
4
8
|
from biolib.app import BioLibApp as _BioLibApp
|
5
9
|
|
10
|
+
# Classes to expose as public API
|
11
|
+
Runtime = _Runtime
|
12
|
+
|
13
|
+
|
6
14
|
def push_app_version(uri: str, path: str) -> _BioLibApp:
|
7
15
|
push_data = _push_application(
|
8
16
|
app_uri=uri,
|
9
17
|
app_path=path,
|
10
18
|
app_version_to_copy_images_from=None,
|
11
|
-
is_dev_version=True
|
19
|
+
is_dev_version=True,
|
20
|
+
)
|
12
21
|
uri = f'{push_data["app_uri"]}:{push_data["sematic_version"]}'
|
13
22
|
return _BioLibApp(uri)
|
14
23
|
|
24
|
+
|
15
25
|
def set_app_version_as_default(app_version: _BioLibApp) -> None:
|
16
26
|
app_version_uuid = app_version.version['public_id']
|
17
27
|
_set_app_version_as_active(app_version_uuid)
|
18
28
|
|
29
|
+
|
19
30
|
def get_app_version_pytest_plugin(app_version: _BioLibApp):
|
20
31
|
try:
|
21
|
-
import pytest
|
32
|
+
import pytest # type: ignore # pylint: disable=import-outside-toplevel,import-error
|
22
33
|
except BaseException:
|
23
34
|
raise Exception('Failed to import pytest; please make sure it is installed') from None
|
24
35
|
|
25
|
-
class AppVersionFixturePlugin
|
36
|
+
class AppVersionFixturePlugin:
|
26
37
|
def __init__(self, app_version_ref):
|
27
38
|
self.app_version_ref = app_version_ref
|
28
39
|
|
29
40
|
@pytest.fixture(scope='session')
|
30
|
-
def app_version(self, request):
|
41
|
+
def app_version(self, request): # pylint: disable=unused-argument
|
31
42
|
return self.app_version_ref
|
32
43
|
|
33
44
|
return AppVersionFixturePlugin(app_version)
|
45
|
+
|
46
|
+
|
47
|
+
def create_data_record(
|
48
|
+
destination: str,
|
49
|
+
data_path: str,
|
50
|
+
name: Optional[str] = None,
|
51
|
+
record_type: Optional[str] = None,
|
52
|
+
) -> _DataRecord:
|
53
|
+
return _DataRecord.create(
|
54
|
+
destination=f'{destination}/{name}' if name else destination,
|
55
|
+
data_path=data_path,
|
56
|
+
record_type=record_type,
|
57
|
+
)
|
biolib/typing_utils.py
CHANGED
@@ -1,7 +1,2 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
# import and expose everything from the typing module
|
4
|
-
from typing import * # pylint: disable=wildcard-import, unused-wildcard-import
|
5
|
-
|
6
|
-
if sys.version_info < (3, 8):
|
7
|
-
from typing_extensions import TypedDict, Literal # pylint: disable=unused-import
|
1
|
+
# TODO: Deprecate and later remove this file
|
2
|
+
from biolib._internal.types.typing import * # pylint: disable=wildcard-import, unused-wildcard-import
|
biolib/user/sign_in.py
CHANGED
@@ -1,19 +1,11 @@
|
|
1
1
|
import time
|
2
|
-
import
|
2
|
+
import webbrowser
|
3
3
|
|
4
4
|
from biolib.biolib_api_client import BiolibApiClient
|
5
5
|
from biolib.biolib_api_client.auth import BiolibAuthChallengeApi
|
6
6
|
from biolib.biolib_logging import logger_no_user_data
|
7
7
|
from biolib.utils import IS_RUNNING_IN_NOTEBOOK
|
8
|
-
|
9
|
-
|
10
|
-
def _open_browser_window(url_to_open: str) -> None:
|
11
|
-
from IPython.display import display, Javascript, update_display # type:ignore # pylint: disable=import-error, import-outside-toplevel
|
12
|
-
|
13
|
-
display_id = str(uuid.uuid4())
|
14
|
-
display(Javascript(f'window.open("{url_to_open}");'), display_id=display_id)
|
15
|
-
time.sleep(1)
|
16
|
-
update_display(Javascript(''), display_id=display_id)
|
8
|
+
from biolib._internal.utils import open_browser_window_from_notebook
|
17
9
|
|
18
10
|
|
19
11
|
def sign_out() -> None:
|
@@ -21,12 +13,12 @@ def sign_out() -> None:
|
|
21
13
|
api_client.sign_out()
|
22
14
|
|
23
15
|
|
24
|
-
def sign_in() -> None:
|
25
|
-
|
26
|
-
if api_client.is_signed_in:
|
16
|
+
def sign_in(open_in_default_browser: bool = False) -> None:
|
17
|
+
if not BiolibApiClient.is_reauthentication_needed():
|
27
18
|
logger_no_user_data.info('Already signed in')
|
28
19
|
return
|
29
20
|
|
21
|
+
api_client = BiolibApiClient.get()
|
30
22
|
auth_challenge = BiolibAuthChallengeApi.create_auth_challenge()
|
31
23
|
auth_challenge_token = auth_challenge['token']
|
32
24
|
|
@@ -37,7 +29,11 @@ def sign_in() -> None:
|
|
37
29
|
if IS_RUNNING_IN_NOTEBOOK:
|
38
30
|
print(f'Opening authorization page at: {frontend_sign_in_url}')
|
39
31
|
print('If your browser does not open automatically, click on the link above.')
|
40
|
-
|
32
|
+
open_browser_window_from_notebook(frontend_sign_in_url)
|
33
|
+
elif open_in_default_browser:
|
34
|
+
print(f'Opening authorization page at: {frontend_sign_in_url}')
|
35
|
+
print('If your browser does not open automatically, click on the link above.')
|
36
|
+
webbrowser.open(frontend_sign_in_url)
|
41
37
|
else:
|
42
38
|
print('Please copy and paste the following link into your browser:')
|
43
39
|
print(frontend_sign_in_url)
|
biolib/utils/__init__.py
CHANGED
@@ -4,9 +4,9 @@ import os
|
|
4
4
|
import socket
|
5
5
|
import sys
|
6
6
|
|
7
|
-
from typing import Optional
|
8
7
|
from importlib_metadata import version, PackageNotFoundError
|
9
8
|
|
9
|
+
from biolib.typing_utils import Optional
|
10
10
|
from biolib.utils.seq_util import SeqUtil, SeqUtilRecord
|
11
11
|
from biolib._internal.http_client import HttpClient
|
12
12
|
from biolib.biolib_logging import logger_no_user_data, logger
|
biolib/utils/app_uri.py
CHANGED
@@ -12,17 +12,18 @@ class SemanticVersion(TypedDict):
|
|
12
12
|
|
13
13
|
class AppUriParsed(TypedDict):
|
14
14
|
account_handle_normalized: str
|
15
|
-
app_name_normalized: str
|
15
|
+
app_name_normalized: Optional[str]
|
16
|
+
app_name: Optional[str]
|
16
17
|
resource_name_prefix: Optional[str]
|
17
18
|
version: Optional[SemanticVersion]
|
18
19
|
|
19
20
|
|
20
|
-
def normalize(string):
|
21
|
+
def normalize(string: str) -> str:
|
21
22
|
return string.replace('-', '_').lower()
|
22
23
|
|
23
24
|
|
24
25
|
# Mainly copied from backend
|
25
|
-
def parse_app_uri(uri: str) -> AppUriParsed:
|
26
|
+
def parse_app_uri(uri: str, use_account_as_name_default: bool = True) -> AppUriParsed:
|
26
27
|
uri_regex = r'^(@(?P<resource_name_prefix>[\w._-]+)/)?(?P<account_handle>[\w-]+)(/(?P<app_name>[\w-]+))?' \
|
27
28
|
r'(:(?P<version>(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)))?$'
|
28
29
|
|
@@ -36,12 +37,18 @@ def parse_app_uri(uri: str) -> AppUriParsed:
|
|
36
37
|
app_name: Optional[str] = matches.group('app_name')
|
37
38
|
|
38
39
|
# Default to account_handle if app_name is not supplied
|
39
|
-
|
40
|
+
if app_name:
|
41
|
+
app_name_normalized = normalize(app_name)
|
42
|
+
elif use_account_as_name_default:
|
43
|
+
app_name_normalized = account_handle_normalized
|
44
|
+
else:
|
45
|
+
app_name_normalized = None
|
40
46
|
|
41
47
|
return AppUriParsed(
|
42
48
|
resource_name_prefix=resource_name_prefix.lower() if resource_name_prefix is not None else 'biolib.com',
|
43
49
|
account_handle_normalized=account_handle_normalized,
|
44
50
|
app_name_normalized=app_name_normalized,
|
51
|
+
app_name=app_name if app_name is not None or not use_account_as_name_default else account_handle_normalized,
|
45
52
|
version=None if not matches.group('version') else SemanticVersion(
|
46
53
|
major=int(matches.group('major')),
|
47
54
|
minor=int(matches.group('minor')),
|
biolib/utils/cache_state.py
CHANGED
@@ -10,7 +10,7 @@ from biolib.biolib_errors import BioLibError
|
|
10
10
|
from biolib.biolib_logging import logger_no_user_data
|
11
11
|
from biolib.typing_utils import Optional, Generic, TypeVar
|
12
12
|
|
13
|
-
StateType = TypeVar('StateType')
|
13
|
+
StateType = TypeVar('StateType') # pylint: disable=invalid-name
|
14
14
|
|
15
15
|
|
16
16
|
class CacheStateError(BioLibError):
|
@@ -37,7 +37,7 @@ class CacheState(abc.ABC, Generic[StateType]):
|
|
37
37
|
def _state_lock_path(self) -> str:
|
38
38
|
return f'{self._state_path}.lock'
|
39
39
|
|
40
|
-
def __init__(self):
|
40
|
+
def __init__(self) -> None:
|
41
41
|
self._state: Optional[StateType] = None
|
42
42
|
|
43
43
|
def __enter__(self) -> StateType:
|