pybiolib 1.1.1747__py3-none-any.whl → 1.1.2193__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. biolib/__init__.py +18 -5
  2. biolib/_data_record/data_record.py +278 -0
  3. biolib/_internal/data_record/__init__.py +1 -0
  4. biolib/_internal/data_record/data_record.py +97 -0
  5. biolib/_internal/data_record/remote_storage_endpoint.py +38 -0
  6. biolib/_internal/file_utils.py +77 -0
  7. biolib/_internal/fuse_mount/__init__.py +1 -0
  8. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  9. biolib/_internal/http_client.py +42 -23
  10. biolib/_internal/lfs/__init__.py +1 -0
  11. biolib/_internal/libs/__init__.py +1 -0
  12. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  13. biolib/_internal/push_application.py +22 -37
  14. biolib/_internal/runtime.py +19 -0
  15. biolib/_internal/types/__init__.py +4 -0
  16. biolib/_internal/types/app.py +9 -0
  17. biolib/_internal/types/data_record.py +40 -0
  18. biolib/_internal/types/experiment.py +10 -0
  19. biolib/_internal/types/resource.py +14 -0
  20. biolib/_internal/types/typing.py +7 -0
  21. biolib/_internal/utils/__init__.py +18 -0
  22. biolib/_runtime/runtime.py +80 -0
  23. biolib/api/__init__.py +1 -0
  24. biolib/api/client.py +39 -17
  25. biolib/app/app.py +40 -72
  26. biolib/app/search_apps.py +8 -12
  27. biolib/biolib_api_client/api_client.py +22 -10
  28. biolib/biolib_api_client/app_types.py +2 -1
  29. biolib/biolib_api_client/biolib_app_api.py +1 -1
  30. biolib/biolib_api_client/biolib_job_api.py +6 -0
  31. biolib/biolib_api_client/job_types.py +4 -4
  32. biolib/biolib_api_client/lfs_types.py +8 -2
  33. biolib/biolib_binary_format/remote_endpoints.py +12 -10
  34. biolib/biolib_binary_format/utils.py +41 -4
  35. biolib/cli/__init__.py +6 -2
  36. biolib/cli/auth.py +58 -0
  37. biolib/cli/data_record.py +80 -0
  38. biolib/cli/download_container.py +3 -1
  39. biolib/cli/init.py +1 -0
  40. biolib/cli/lfs.py +45 -11
  41. biolib/cli/push.py +1 -1
  42. biolib/cli/run.py +3 -2
  43. biolib/cli/start.py +1 -0
  44. biolib/compute_node/cloud_utils/cloud_utils.py +15 -18
  45. biolib/compute_node/job_worker/cache_state.py +1 -1
  46. biolib/compute_node/job_worker/executors/docker_executor.py +134 -114
  47. biolib/compute_node/job_worker/job_storage.py +3 -4
  48. biolib/compute_node/job_worker/job_worker.py +31 -15
  49. biolib/compute_node/remote_host_proxy.py +75 -70
  50. biolib/compute_node/webserver/webserver_types.py +0 -1
  51. biolib/experiments/experiment.py +75 -44
  52. biolib/jobs/job.py +125 -47
  53. biolib/jobs/job_result.py +46 -21
  54. biolib/jobs/types.py +1 -1
  55. biolib/runtime/__init__.py +14 -1
  56. biolib/sdk/__init__.py +29 -5
  57. biolib/typing_utils.py +2 -7
  58. biolib/user/sign_in.py +10 -14
  59. biolib/utils/__init__.py +1 -1
  60. biolib/utils/app_uri.py +11 -4
  61. biolib/utils/cache_state.py +2 -2
  62. biolib/utils/seq_util.py +38 -30
  63. {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/METADATA +1 -1
  64. pybiolib-1.1.2193.dist-info/RECORD +123 -0
  65. {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/WHEEL +1 -1
  66. biolib/biolib_api_client/biolib_account_api.py +0 -8
  67. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -34
  68. biolib/experiments/types.py +0 -9
  69. biolib/lfs/__init__.py +0 -6
  70. biolib/lfs/utils.py +0 -237
  71. biolib/runtime/results.py +0 -20
  72. pybiolib-1.1.1747.dist-info/RECORD +0 -108
  73. /biolib/{lfs → _internal/lfs}/cache.py +0 -0
  74. {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/LICENSE +0 -0
  75. {pybiolib-1.1.1747.dist-info → pybiolib-1.1.2193.dist-info}/entry_points.txt +0 -0
biolib/jobs/job.py CHANGED
@@ -1,34 +1,42 @@
1
1
  import base64
2
- from datetime import datetime, timedelta
3
2
  import sys
4
3
  import time
5
- from pathlib import Path
6
4
  from collections import OrderedDict
5
+ from datetime import datetime, timedelta
6
+ from pathlib import Path
7
7
  from urllib.parse import urlparse
8
8
 
9
9
  from biolib import api, utils
10
10
  from biolib._internal.http_client import HttpClient
11
+ from biolib._internal.utils import open_browser_window_from_notebook
12
+ from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
13
+ from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
11
14
  from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
12
- from biolib.biolib_binary_format import LazyLoadedFile, ModuleOutputV2, ModuleInput, ModuleInputDict
15
+ from biolib.biolib_binary_format import LazyLoadedFile, ModuleInput, ModuleInputDict, ModuleOutputV2
16
+ from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
13
17
  from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
14
18
  from biolib.biolib_errors import BioLibError, CloudJobFinishedError
15
19
  from biolib.biolib_logging import logger, logger_no_user_data
20
+ from biolib.compute_node.job_worker.job_storage import JobStorage
16
21
  from biolib.compute_node.utils import SystemExceptionCodeMap, SystemExceptionCodes
17
22
  from biolib.jobs.job_result import JobResult
18
- from biolib.jobs.types import JobDict, CloudJobStartedDict, CloudJobDict
23
+ from biolib.jobs.types import CloudJobDict, CloudJobStartedDict, JobDict
19
24
  from biolib.tables import BioLibTable
20
- from biolib.typing_utils import Optional, List, cast, Dict
25
+ from biolib.typing_utils import Dict, List, Optional, cast
21
26
  from biolib.utils import IS_RUNNING_IN_NOTEBOOK
27
+ from biolib.utils.app_uri import parse_app_uri
22
28
 
23
29
 
24
30
  class Job:
25
31
  # Columns to print in table when showing Job
26
- table_columns_to_row_map = OrderedDict({
27
- 'ID': {'key': 'uuid', 'params': {'width': 36}},
28
- 'Application': {'key': 'app_uri', 'params': {}},
29
- 'Status': {'key': 'state', 'params': {}},
30
- 'Started At': {'key': 'started_at', 'params': {}},
31
- })
32
+ table_columns_to_row_map = OrderedDict(
33
+ {
34
+ 'ID': {'key': 'uuid', 'params': {'width': 36}},
35
+ 'Application': {'key': 'app_uri', 'params': {}},
36
+ 'Status': {'key': 'state', 'params': {}},
37
+ 'Started At': {'key': 'started_at', 'params': {}},
38
+ }
39
+ )
32
40
 
33
41
  def __init__(self, job_dict: JobDict):
34
42
  self._uuid: str = job_dict['uuid']
@@ -52,26 +60,23 @@ class Job:
52
60
  @property
53
61
  def result(self) -> JobResult:
54
62
  if not self._result:
55
- if self.get_status() == "completed":
56
- self._result = JobResult(job_uuid=self._uuid, job_auth_token=self._auth_token)
57
- else:
58
- raise BioLibError(f"Result is not available for {self._uuid}: status is {self._job_dict['state']}.")
63
+ self._result = JobResult(job_uuid=self._uuid, job_auth_token=self._auth_token)
59
64
 
60
65
  return self._result
61
66
 
62
67
  @property
63
68
  def stdout(self) -> bytes:
64
- logger.warning("The property .stdout is deprecated, please use .get_stdout()")
69
+ logger.warning('The property .stdout is deprecated, please use .get_stdout()')
65
70
  return self.result.get_stdout()
66
71
 
67
72
  @property
68
73
  def stderr(self) -> bytes:
69
- logger.warning("The property .stderr is deprecated, please use .get_stderr()")
74
+ logger.warning('The property .stderr is deprecated, please use .get_stderr()')
70
75
  return self.result.get_stderr()
71
76
 
72
77
  @property
73
78
  def exitcode(self) -> int:
74
- logger.warning("The property .exitcode is deprecated, please use .get_exit_code()")
79
+ logger.warning('The property .exitcode is deprecated, please use .get_exit_code()')
75
80
  return self.result.get_exit_code()
76
81
 
77
82
  def is_finished(self) -> bool:
@@ -105,8 +110,8 @@ class Job:
105
110
  def load_file_as_numpy(self, *args, **kwargs):
106
111
  try:
107
112
  import numpy # type: ignore # pylint: disable=import-outside-toplevel,import-error
108
- except: # pylint: disable=raise-missing-from
109
- raise Exception("Failed to import numpy, please make sure it is installed.")
113
+ except ImportError: # pylint: disable=raise-missing-from
114
+ raise Exception('Failed to import numpy, please make sure it is installed.') from None
110
115
  file_handle = self.result.get_output_file(*args, **kwargs).get_file_handle()
111
116
  return numpy.load(file_handle, allow_pickle=False) # type: ignore
112
117
 
@@ -172,6 +177,50 @@ class Job:
172
177
  time.sleep(2)
173
178
  logger.info(f'Job {self.id} has finished.')
174
179
 
180
+ def open_browser(self) -> None:
181
+ api_client = BiolibApiClient.get()
182
+ results_url_to_open = f'{api_client.base_url}/results/{self.id}/?token={self._auth_token}'
183
+ if IS_RUNNING_IN_NOTEBOOK:
184
+ print(f'Opening results page at: {results_url_to_open}')
185
+ print('If your browser does not open automatically, click on the link above.')
186
+ open_browser_window_from_notebook(results_url_to_open)
187
+ else:
188
+ print('Please copy and paste the following link into your browser:')
189
+ print(results_url_to_open)
190
+
191
+ def cancel(self) -> None:
192
+ try:
193
+ api.client.patch(
194
+ path=f'/jobs/{self._uuid}/',
195
+ headers={'Job-Auth-Token': self._auth_token} if self._auth_token else None,
196
+ data={'state': 'cancelled'},
197
+ )
198
+ logger.info(f'Job {self._uuid} canceled')
199
+ except Exception as error:
200
+ logger.error(f'Failed to cancel job {self._uuid} due to: {error}')
201
+
202
+ def recompute(self, app_uri: Optional[str] = None, machine: Optional[str] = None, blocking: bool = True) -> 'Job':
203
+ app_response = BiolibAppApi.get_by_uri(uri=app_uri or self._job_dict['app_uri'])
204
+
205
+ job_storage_input = RemoteJobStorageEndpoint(
206
+ job_auth_token=self._auth_token,
207
+ job_uuid=self._uuid,
208
+ storage_type='input',
209
+ )
210
+ http_response = HttpClient.request(url=job_storage_input.get_remote_url())
211
+ module_input_serialized = http_response.content
212
+
213
+ job = self._start_job_in_cloud(
214
+ app_uri=app_response['app_uri'],
215
+ app_version_uuid=app_response['app_version']['public_id'],
216
+ module_input_serialized=module_input_serialized,
217
+ machine=machine,
218
+ )
219
+ if blocking:
220
+ job.stream_logs()
221
+
222
+ return job
223
+
175
224
  def _get_cloud_job(self) -> CloudJobDict:
176
225
  self._refetch_job_dict(force_refetch=True)
177
226
  if self._job_dict['cloud_job'] is None:
@@ -190,20 +239,11 @@ class Job:
190
239
  @staticmethod
191
240
  def show_jobs(count: int = 25) -> None:
192
241
  job_dicts = Job._get_job_dicts(count)
193
- BioLibTable(
194
- columns_to_row_map=Job.table_columns_to_row_map,
195
- rows=job_dicts,
196
- title='Jobs'
197
- ).print_table()
242
+ BioLibTable(columns_to_row_map=Job.table_columns_to_row_map, rows=job_dicts, title='Jobs').print_table()
198
243
 
199
244
  @staticmethod
200
245
  def _get_job_dicts(count: int) -> List['JobDict']:
201
- job_dicts: List['JobDict'] = api.client.get(
202
- path='/jobs/',
203
- params={
204
- 'page_size': str(count)
205
- }
206
- ).json()['results']
246
+ job_dicts: List['JobDict'] = api.client.get(path='/jobs/', params={'page_size': str(count)}).json()['results']
207
247
  return job_dicts
208
248
 
209
249
  @staticmethod
@@ -235,9 +275,7 @@ class Job:
235
275
  def show(self) -> None:
236
276
  self._refetch_job_dict()
237
277
  BioLibTable(
238
- columns_to_row_map=Job.table_columns_to_row_map,
239
- rows=[self._job_dict],
240
- title=f'Job: {self._uuid}'
278
+ columns_to_row_map=Job.table_columns_to_row_map, rows=[self._job_dict], title=f'Job: {self._uuid}'
241
279
  ).print_table()
242
280
 
243
281
  def stream_logs(self) -> None:
@@ -274,7 +312,7 @@ class Job:
274
312
  status_json = self._get_job_status_from_compute_node(compute_node_url)
275
313
  if not status_json:
276
314
  # this can happen if the job is finished but already removed from the compute node
277
- logger.warning("WARN: We were unable to retrieve the full log of the job, please try again")
315
+ logger.warning('WARN: We were unable to retrieve the full log of the job, please try again')
278
316
  break
279
317
  job_is_completed = status_json['is_completed']
280
318
  for status_update in status_json['status_updates']:
@@ -305,12 +343,10 @@ class Job:
305
343
 
306
344
  def _print_full_logs(self, node_url: str) -> None:
307
345
  try:
308
- response_json = HttpClient.request(
309
- url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full'
310
- ).json()
346
+ response_json = HttpClient.request(url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full').json()
311
347
  except Exception as error:
312
348
  logger.error(f'Could not get full streamed logs due to: {error}')
313
- raise BioLibError from error
349
+ raise BioLibError('Could not get full streamed logs') from error
314
350
 
315
351
  for status_update in response_json.get('previous_status_updates', []):
316
352
  logger.info(f'Cloud: {status_update["log_message"]}')
@@ -318,7 +354,10 @@ class Job:
318
354
  self.print_logs_packages(response_json['streamed_logs_packages_b64'])
319
355
 
320
356
  def _get_cloud_job_awaiting_started(self) -> CloudJobStartedDict:
357
+ retry_count = 0
321
358
  while True:
359
+ retry_count += 1
360
+ time.sleep(min(10, retry_count))
322
361
  cloud_job = self._get_cloud_job()
323
362
 
324
363
  if cloud_job['finished_at']:
@@ -331,23 +370,19 @@ class Job:
331
370
  return cast(CloudJobStartedDict, cloud_job)
332
371
 
333
372
  logger.info('Cloud: The job has been queued. Please wait...')
334
- time.sleep(10)
335
373
 
336
374
  def _get_job_status_from_compute_node(self, compute_node_url):
337
375
  for _ in range(15):
338
376
  try:
339
- return HttpClient.request(
340
- url=f'{compute_node_url}/v1/job/{self._uuid}/status/'
341
- ).json()
377
+ return HttpClient.request(url=f'{compute_node_url}/v1/job/{self._uuid}/status/').json()
342
378
  except Exception: # pylint: disable=broad-except
343
379
  cloud_job = self._get_cloud_job()
344
- logger.debug("Failed to get status from compute node, retrying...")
380
+ logger.debug('Failed to get status from compute node, retrying...')
345
381
  if cloud_job['finished_at']:
346
- logger.debug("Job no longer exists on compute node, checking for error...")
382
+ logger.debug('Job no longer exists on compute node, checking for error...')
347
383
  if cloud_job['error_code'] != SystemExceptionCodes.COMPLETED_SUCCESSFULLY.value:
348
384
  error_message = SystemExceptionCodeMap.get(
349
- cloud_job['error_code'],
350
- f'Unknown error code {cloud_job["error_code"]}'
385
+ cloud_job['error_code'], f'Unknown error code {cloud_job["error_code"]}'
351
386
  )
352
387
  raise BioLibError(f'Cloud: {error_message}') from None
353
388
  else:
@@ -367,3 +402,46 @@ class Job:
367
402
 
368
403
  self._job_dict = self._get_job_dict(self._uuid, self._auth_token)
369
404
  self._job_dict_last_fetched_at = datetime.utcnow()
405
+
406
+ @staticmethod
407
+ def _start_job_in_cloud(
408
+ app_uri: str,
409
+ app_version_uuid: str,
410
+ module_input_serialized: bytes,
411
+ override_command: bool = False,
412
+ machine: Optional[str] = None,
413
+ experiment_id: Optional[str] = None,
414
+ result_prefix: Optional[str] = None,
415
+ timeout: Optional[int] = None,
416
+ notify: bool = False,
417
+ requested_machine_count: Optional[int] = None,
418
+ ) -> 'Job':
419
+ if len(module_input_serialized) < 500_000:
420
+ _job_dict = BiolibJobApi.create_job_with_data(
421
+ app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
422
+ app_version_uuid=app_version_uuid,
423
+ arguments_override_command=override_command,
424
+ experiment_uuid=experiment_id,
425
+ module_input_serialized=module_input_serialized,
426
+ notify=notify,
427
+ requested_machine=machine,
428
+ requested_timeout_seconds=timeout,
429
+ result_name_prefix=result_prefix,
430
+ requested_machine_count=requested_machine_count,
431
+ )
432
+ return Job(cast(JobDict, _job_dict))
433
+
434
+ job_dict: CreatedJobDict = BiolibJobApi.create(
435
+ app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
436
+ app_version_id=app_version_uuid,
437
+ experiment_uuid=experiment_id,
438
+ machine=machine,
439
+ notify=notify,
440
+ override_command=override_command,
441
+ timeout=timeout,
442
+ requested_machine_count=requested_machine_count,
443
+ )
444
+ JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
445
+ cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
446
+ logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
447
+ return Job(cast(JobDict, job_dict))
biolib/jobs/job_result.py CHANGED
@@ -1,25 +1,24 @@
1
- from pathlib import Path
2
- from fnmatch import fnmatch
3
1
  import time
2
+ from fnmatch import fnmatch
3
+ from pathlib import Path
4
4
 
5
5
  from biolib.biolib_binary_format import ModuleOutputV2
6
+ from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
6
7
  from biolib.biolib_binary_format.remote_stream_seeker import StreamSeeker
7
- from biolib.biolib_binary_format.utils import RemoteIndexableBuffer, LazyLoadedFile
8
- from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageResultEndpoint
8
+ from biolib.biolib_binary_format.utils import LazyLoadedFile, RemoteIndexableBuffer
9
9
  from biolib.biolib_errors import BioLibError
10
10
  from biolib.biolib_logging import logger
11
- from biolib.typing_utils import Optional, List, cast, Union, Callable
11
+ from biolib.typing_utils import Callable, List, Optional, Union, cast
12
12
 
13
13
  PathFilter = Union[str, Callable[[str], bool]]
14
14
 
15
15
 
16
16
  class JobResult:
17
-
18
17
  def __init__(
19
- self,
20
- job_uuid: str,
21
- job_auth_token: str,
22
- module_output: Optional[ModuleOutputV2] = None,
18
+ self,
19
+ job_uuid: str,
20
+ job_auth_token: str,
21
+ module_output: Optional[ModuleOutputV2] = None,
23
22
  ):
24
23
  self._job_uuid: str = job_uuid
25
24
  self._job_auth_token: str = job_auth_token
@@ -35,7 +34,12 @@ class JobResult:
35
34
  def get_exit_code(self) -> int:
36
35
  return self._get_module_output().get_exit_code()
37
36
 
38
- def save_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
37
+ def save_files(
38
+ self,
39
+ output_dir: str,
40
+ path_filter: Optional[PathFilter] = None,
41
+ skip_file_if_exists: Optional[bool] = None,
42
+ ) -> None:
39
43
  module_output = self._get_module_output()
40
44
  output_files = module_output.get_files()
41
45
  filtered_output_files = self._get_filtered_files(output_files, path_filter) if path_filter else output_files
@@ -61,24 +65,44 @@ class JobResult:
61
65
  # Remove leading slash of file_path
62
66
  destination_file_path = Path(output_dir) / Path(file.path.lstrip('/'))
63
67
  if destination_file_path.exists():
64
- destination_file_path.rename(f'{destination_file_path}.biolib-renamed.{time.strftime("%Y%m%d%H%M%S")}')
68
+ if skip_file_if_exists:
69
+ print(f'Skipping {destination_file_path} as a file with that name already exists locally.')
70
+ continue
71
+ else:
72
+ destination_file_path.rename(
73
+ f'{destination_file_path}.biolib-renamed.{time.strftime("%Y%m%d%H%M%S")}'
74
+ )
65
75
 
66
76
  dir_path = destination_file_path.parent
67
77
  if dir_path:
68
78
  dir_path.mkdir(parents=True, exist_ok=True)
69
79
 
70
- with open(destination_file_path, mode='wb') as destination_file:
71
- for chunk in stream_seeker.seek_and_read(file_start=file.start, file_length=file.length):
72
- destination_file.write(chunk)
80
+ # write content to temporary (partial) file
81
+ partial_path = destination_file_path.with_suffix(
82
+ destination_file_path.suffix + f'.{self._job_uuid}.partial_biolib_download'
83
+ )
84
+ file_start = file.start
85
+ data_to_download = file.length
86
+ if partial_path.exists():
87
+ data_already_downloaded = partial_path.stat().st_size
88
+ file_start += data_already_downloaded
89
+ data_to_download -= data_already_downloaded
90
+
91
+ with open(partial_path, mode='ab') as partial_file:
92
+ for chunk in stream_seeker.seek_and_read(file_start=file_start, file_length=data_to_download):
93
+ partial_file.write(chunk)
94
+
95
+ # rename partial file to actual file name
96
+ partial_path.rename(destination_file_path)
73
97
 
74
98
  def get_output_file(self, filename) -> LazyLoadedFile:
75
99
  files = self._get_module_output().get_files()
76
100
  filtered_files = self._get_filtered_files(files, path_filter=filename)
77
101
  if not filtered_files:
78
- raise BioLibError(f"File {filename} not found in results.")
102
+ raise BioLibError(f'File {filename} not found in results.')
79
103
 
80
104
  if len(filtered_files) != 1:
81
- raise BioLibError(f"Found multiple results for filename {filename}.")
105
+ raise BioLibError(f'Found multiple results for filename {filename}.')
82
106
 
83
107
  return filtered_files[0]
84
108
 
@@ -100,8 +124,8 @@ class JobResult:
100
124
  glob_filter = cast(str, path_filter)
101
125
 
102
126
  # since all file paths start with /, make sure filter does too
103
- if not glob_filter.startswith("/"):
104
- glob_filter = "/" + glob_filter
127
+ if not glob_filter.startswith('/'):
128
+ glob_filter = '/' + glob_filter
105
129
 
106
130
  def _filter_function(file: LazyLoadedFile) -> bool:
107
131
  return fnmatch(file.path, glob_filter)
@@ -110,9 +134,10 @@ class JobResult:
110
134
 
111
135
  def _get_module_output(self) -> ModuleOutputV2:
112
136
  if self._module_output is None:
113
- remote_job_storage_endpoint = RemoteJobStorageResultEndpoint(
114
- job_id=self._job_uuid,
137
+ remote_job_storage_endpoint = RemoteJobStorageEndpoint(
115
138
  job_auth_token=self._job_auth_token,
139
+ job_uuid=self._job_uuid,
140
+ storage_type='output',
116
141
  )
117
142
  buffer = RemoteIndexableBuffer(endpoint=remote_job_storage_endpoint)
118
143
  self._module_output = ModuleOutputV2(buffer)
biolib/jobs/types.py CHANGED
@@ -1,4 +1,4 @@
1
- from biolib.typing_utils import TypedDict, Optional, Literal, List
1
+ from biolib.typing_utils import List, Literal, Optional, TypedDict
2
2
 
3
3
  JobState = Literal['in_progress', 'completed', 'failed', 'cancelled']
4
4
 
@@ -1 +1,14 @@
1
- from .results import set_main_result_prefix
1
+ import warnings
2
+
3
+ from biolib._runtime.runtime import Runtime as _Runtime
4
+
5
+
6
+ def set_main_result_prefix(result_prefix: str) -> None:
7
+ warnings.warn(
8
+ 'The "biolib.runtime.set_main_result_prefix" function is deprecated. '
9
+ 'It will be removed in future releases from mid 2024. '
10
+ 'Please use "from biolib.sdk import Runtime" and then "Runtime.set_main_result_prefix" instead.',
11
+ DeprecationWarning,
12
+ stacklevel=2,
13
+ )
14
+ _Runtime.set_main_result_prefix(result_prefix)
biolib/sdk/__init__.py CHANGED
@@ -1,33 +1,57 @@
1
+ from typing import Optional
2
+
3
+ # Imports to hide and use as private internal utils
4
+ from biolib._data_record.data_record import DataRecord as _DataRecord
1
5
  from biolib._internal.push_application import push_application as _push_application
2
6
  from biolib._internal.push_application import set_app_version_as_active as _set_app_version_as_active
3
-
7
+ from biolib._runtime.runtime import Runtime as _Runtime
4
8
  from biolib.app import BioLibApp as _BioLibApp
5
9
 
10
+ # Classes to expose as public API
11
+ Runtime = _Runtime
12
+
13
+
6
14
  def push_app_version(uri: str, path: str) -> _BioLibApp:
7
15
  push_data = _push_application(
8
16
  app_uri=uri,
9
17
  app_path=path,
10
18
  app_version_to_copy_images_from=None,
11
- is_dev_version=True)
19
+ is_dev_version=True,
20
+ )
12
21
  uri = f'{push_data["app_uri"]}:{push_data["sematic_version"]}'
13
22
  return _BioLibApp(uri)
14
23
 
24
+
15
25
  def set_app_version_as_default(app_version: _BioLibApp) -> None:
16
26
  app_version_uuid = app_version.version['public_id']
17
27
  _set_app_version_as_active(app_version_uuid)
18
28
 
29
+
19
30
  def get_app_version_pytest_plugin(app_version: _BioLibApp):
20
31
  try:
21
- import pytest # type: ignore # pylint: disable=import-outside-toplevel,import-error
32
+ import pytest # type: ignore # pylint: disable=import-outside-toplevel,import-error
22
33
  except BaseException:
23
34
  raise Exception('Failed to import pytest; please make sure it is installed') from None
24
35
 
25
- class AppVersionFixturePlugin(object):
36
+ class AppVersionFixturePlugin:
26
37
  def __init__(self, app_version_ref):
27
38
  self.app_version_ref = app_version_ref
28
39
 
29
40
  @pytest.fixture(scope='session')
30
- def app_version(self, request): # pylint: disable=unused-argument
41
+ def app_version(self, request): # pylint: disable=unused-argument
31
42
  return self.app_version_ref
32
43
 
33
44
  return AppVersionFixturePlugin(app_version)
45
+
46
+
47
+ def create_data_record(
48
+ destination: str,
49
+ data_path: str,
50
+ name: Optional[str] = None,
51
+ record_type: Optional[str] = None,
52
+ ) -> _DataRecord:
53
+ return _DataRecord.create(
54
+ destination=f'{destination}/{name}' if name else destination,
55
+ data_path=data_path,
56
+ record_type=record_type,
57
+ )
biolib/typing_utils.py CHANGED
@@ -1,7 +1,2 @@
1
- import sys
2
-
3
- # import and expose everything from the typing module
4
- from typing import * # pylint: disable=wildcard-import, unused-wildcard-import
5
-
6
- if sys.version_info < (3, 8):
7
- from typing_extensions import TypedDict, Literal # pylint: disable=unused-import
1
+ # TODO: Deprecate and later remove this file
2
+ from biolib._internal.types.typing import * # pylint: disable=wildcard-import, unused-wildcard-import
biolib/user/sign_in.py CHANGED
@@ -1,19 +1,11 @@
1
1
  import time
2
- import uuid
2
+ import webbrowser
3
3
 
4
4
  from biolib.biolib_api_client import BiolibApiClient
5
5
  from biolib.biolib_api_client.auth import BiolibAuthChallengeApi
6
6
  from biolib.biolib_logging import logger_no_user_data
7
7
  from biolib.utils import IS_RUNNING_IN_NOTEBOOK
8
-
9
-
10
- def _open_browser_window(url_to_open: str) -> None:
11
- from IPython.display import display, Javascript, update_display # type:ignore # pylint: disable=import-error, import-outside-toplevel
12
-
13
- display_id = str(uuid.uuid4())
14
- display(Javascript(f'window.open("{url_to_open}");'), display_id=display_id)
15
- time.sleep(1)
16
- update_display(Javascript(''), display_id=display_id)
8
+ from biolib._internal.utils import open_browser_window_from_notebook
17
9
 
18
10
 
19
11
  def sign_out() -> None:
@@ -21,12 +13,12 @@ def sign_out() -> None:
21
13
  api_client.sign_out()
22
14
 
23
15
 
24
- def sign_in() -> None:
25
- api_client = BiolibApiClient.get()
26
- if api_client.is_signed_in:
16
+ def sign_in(open_in_default_browser: bool = False) -> None:
17
+ if not BiolibApiClient.is_reauthentication_needed():
27
18
  logger_no_user_data.info('Already signed in')
28
19
  return
29
20
 
21
+ api_client = BiolibApiClient.get()
30
22
  auth_challenge = BiolibAuthChallengeApi.create_auth_challenge()
31
23
  auth_challenge_token = auth_challenge['token']
32
24
 
@@ -37,7 +29,11 @@ def sign_in() -> None:
37
29
  if IS_RUNNING_IN_NOTEBOOK:
38
30
  print(f'Opening authorization page at: {frontend_sign_in_url}')
39
31
  print('If your browser does not open automatically, click on the link above.')
40
- _open_browser_window(frontend_sign_in_url)
32
+ open_browser_window_from_notebook(frontend_sign_in_url)
33
+ elif open_in_default_browser:
34
+ print(f'Opening authorization page at: {frontend_sign_in_url}')
35
+ print('If your browser does not open automatically, click on the link above.')
36
+ webbrowser.open(frontend_sign_in_url)
41
37
  else:
42
38
  print('Please copy and paste the following link into your browser:')
43
39
  print(frontend_sign_in_url)
biolib/utils/__init__.py CHANGED
@@ -4,9 +4,9 @@ import os
4
4
  import socket
5
5
  import sys
6
6
 
7
- from typing import Optional
8
7
  from importlib_metadata import version, PackageNotFoundError
9
8
 
9
+ from biolib.typing_utils import Optional
10
10
  from biolib.utils.seq_util import SeqUtil, SeqUtilRecord
11
11
  from biolib._internal.http_client import HttpClient
12
12
  from biolib.biolib_logging import logger_no_user_data, logger
biolib/utils/app_uri.py CHANGED
@@ -12,17 +12,18 @@ class SemanticVersion(TypedDict):
12
12
 
13
13
  class AppUriParsed(TypedDict):
14
14
  account_handle_normalized: str
15
- app_name_normalized: str
15
+ app_name_normalized: Optional[str]
16
+ app_name: Optional[str]
16
17
  resource_name_prefix: Optional[str]
17
18
  version: Optional[SemanticVersion]
18
19
 
19
20
 
20
- def normalize(string):
21
+ def normalize(string: str) -> str:
21
22
  return string.replace('-', '_').lower()
22
23
 
23
24
 
24
25
  # Mainly copied from backend
25
- def parse_app_uri(uri: str) -> AppUriParsed:
26
+ def parse_app_uri(uri: str, use_account_as_name_default: bool = True) -> AppUriParsed:
26
27
  uri_regex = r'^(@(?P<resource_name_prefix>[\w._-]+)/)?(?P<account_handle>[\w-]+)(/(?P<app_name>[\w-]+))?' \
27
28
  r'(:(?P<version>(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)))?$'
28
29
 
@@ -36,12 +37,18 @@ def parse_app_uri(uri: str) -> AppUriParsed:
36
37
  app_name: Optional[str] = matches.group('app_name')
37
38
 
38
39
  # Default to account_handle if app_name is not supplied
39
- app_name_normalized = normalize(app_name) if app_name is not None else account_handle_normalized
40
+ if app_name:
41
+ app_name_normalized = normalize(app_name)
42
+ elif use_account_as_name_default:
43
+ app_name_normalized = account_handle_normalized
44
+ else:
45
+ app_name_normalized = None
40
46
 
41
47
  return AppUriParsed(
42
48
  resource_name_prefix=resource_name_prefix.lower() if resource_name_prefix is not None else 'biolib.com',
43
49
  account_handle_normalized=account_handle_normalized,
44
50
  app_name_normalized=app_name_normalized,
51
+ app_name=app_name if app_name is not None or not use_account_as_name_default else account_handle_normalized,
45
52
  version=None if not matches.group('version') else SemanticVersion(
46
53
  major=int(matches.group('major')),
47
54
  minor=int(matches.group('minor')),
@@ -10,7 +10,7 @@ from biolib.biolib_errors import BioLibError
10
10
  from biolib.biolib_logging import logger_no_user_data
11
11
  from biolib.typing_utils import Optional, Generic, TypeVar
12
12
 
13
- StateType = TypeVar('StateType')
13
+ StateType = TypeVar('StateType') # pylint: disable=invalid-name
14
14
 
15
15
 
16
16
  class CacheStateError(BioLibError):
@@ -37,7 +37,7 @@ class CacheState(abc.ABC, Generic[StateType]):
37
37
  def _state_lock_path(self) -> str:
38
38
  return f'{self._state_path}.lock'
39
39
 
40
- def __init__(self):
40
+ def __init__(self) -> None:
41
41
  self._state: Optional[StateType] = None
42
42
 
43
43
  def __enter__(self) -> StateType: