pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. biolib/__init__.py +357 -11
  2. biolib/_data_record/data_record.py +380 -0
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +55 -0
  5. biolib/_index/query_result.py +103 -0
  6. biolib/_internal/__init__.py +0 -0
  7. biolib/_internal/add_copilot_prompts.py +58 -0
  8. biolib/_internal/add_gui_files.py +81 -0
  9. biolib/_internal/data_record/__init__.py +1 -0
  10. biolib/_internal/data_record/data_record.py +85 -0
  11. biolib/_internal/data_record/push_data.py +116 -0
  12. biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
  13. biolib/_internal/errors.py +5 -0
  14. biolib/_internal/file_utils.py +125 -0
  15. biolib/_internal/fuse_mount/__init__.py +1 -0
  16. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  17. biolib/_internal/http_client.py +159 -0
  18. biolib/_internal/lfs/__init__.py +1 -0
  19. biolib/_internal/lfs/cache.py +51 -0
  20. biolib/_internal/libs/__init__.py +1 -0
  21. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  22. biolib/_internal/push_application.py +488 -0
  23. biolib/_internal/runtime.py +22 -0
  24. biolib/_internal/string_utils.py +13 -0
  25. biolib/_internal/templates/__init__.py +1 -0
  26. biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
  27. biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
  28. biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
  29. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  30. biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
  31. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  32. biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
  33. biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
  34. biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
  35. biolib/_internal/templates/gitignore_template/.gitignore +10 -0
  36. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  37. biolib/_internal/templates/gui_template/App.tsx +53 -0
  38. biolib/_internal/templates/gui_template/Dockerfile +27 -0
  39. biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
  40. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  41. biolib/_internal/templates/gui_template/index.css +5 -0
  42. biolib/_internal/templates/gui_template/index.html +13 -0
  43. biolib/_internal/templates/gui_template/index.tsx +10 -0
  44. biolib/_internal/templates/gui_template/package.json +27 -0
  45. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  46. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
  47. biolib/_internal/templates/gui_template/vite.config.mts +10 -0
  48. biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
  49. biolib/_internal/templates/init_template/Dockerfile +14 -0
  50. biolib/_internal/templates/init_template/requirements.txt +1 -0
  51. biolib/_internal/templates/init_template/run.py +12 -0
  52. biolib/_internal/templates/init_template/run.sh +4 -0
  53. biolib/_internal/templates/templates.py +25 -0
  54. biolib/_internal/tree_utils.py +106 -0
  55. biolib/_internal/utils/__init__.py +65 -0
  56. biolib/_internal/utils/auth.py +46 -0
  57. biolib/_internal/utils/job_url.py +33 -0
  58. biolib/_internal/utils/multinode.py +263 -0
  59. biolib/_runtime/runtime.py +157 -0
  60. biolib/_session/session.py +44 -0
  61. biolib/_shared/__init__.py +0 -0
  62. biolib/_shared/types/__init__.py +74 -0
  63. biolib/_shared/types/account.py +12 -0
  64. biolib/_shared/types/account_member.py +8 -0
  65. biolib/_shared/types/app.py +9 -0
  66. biolib/_shared/types/data_record.py +40 -0
  67. biolib/_shared/types/experiment.py +32 -0
  68. biolib/_shared/types/file_node.py +17 -0
  69. biolib/_shared/types/push.py +6 -0
  70. biolib/_shared/types/resource.py +37 -0
  71. biolib/_shared/types/resource_deploy_key.py +11 -0
  72. biolib/_shared/types/resource_permission.py +14 -0
  73. biolib/_shared/types/resource_version.py +19 -0
  74. biolib/_shared/types/result.py +14 -0
  75. biolib/_shared/types/typing.py +10 -0
  76. biolib/_shared/types/user.py +19 -0
  77. biolib/_shared/utils/__init__.py +7 -0
  78. biolib/_shared/utils/resource_uri.py +75 -0
  79. biolib/api/__init__.py +6 -0
  80. biolib/api/client.py +168 -0
  81. biolib/app/app.py +252 -49
  82. biolib/app/search_apps.py +45 -0
  83. biolib/biolib_api_client/api_client.py +126 -31
  84. biolib/biolib_api_client/app_types.py +24 -4
  85. biolib/biolib_api_client/auth.py +31 -8
  86. biolib/biolib_api_client/biolib_app_api.py +147 -52
  87. biolib/biolib_api_client/biolib_job_api.py +161 -141
  88. biolib/biolib_api_client/job_types.py +21 -5
  89. biolib/biolib_api_client/lfs_types.py +7 -23
  90. biolib/biolib_api_client/user_state.py +56 -0
  91. biolib/biolib_binary_format/__init__.py +1 -4
  92. biolib/biolib_binary_format/file_in_container.py +105 -0
  93. biolib/biolib_binary_format/module_input.py +24 -7
  94. biolib/biolib_binary_format/module_output_v2.py +149 -0
  95. biolib/biolib_binary_format/remote_endpoints.py +34 -0
  96. biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
  97. biolib/biolib_binary_format/saved_job.py +3 -2
  98. biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
  99. biolib/biolib_binary_format/system_status_update.py +3 -2
  100. biolib/biolib_binary_format/utils.py +175 -0
  101. biolib/biolib_docker_client/__init__.py +11 -2
  102. biolib/biolib_errors.py +36 -0
  103. biolib/biolib_logging.py +27 -10
  104. biolib/cli/__init__.py +38 -0
  105. biolib/cli/auth.py +46 -0
  106. biolib/cli/data_record.py +164 -0
  107. biolib/cli/index.py +32 -0
  108. biolib/cli/init.py +421 -0
  109. biolib/cli/lfs.py +101 -0
  110. biolib/cli/push.py +50 -0
  111. biolib/cli/run.py +63 -0
  112. biolib/cli/runtime.py +14 -0
  113. biolib/cli/sdk.py +16 -0
  114. biolib/cli/start.py +56 -0
  115. biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
  116. biolib/compute_node/job_worker/cache_state.py +66 -88
  117. biolib/compute_node/job_worker/cache_types.py +1 -6
  118. biolib/compute_node/job_worker/docker_image_cache.py +112 -37
  119. biolib/compute_node/job_worker/executors/__init__.py +0 -3
  120. biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
  121. biolib/compute_node/job_worker/executors/docker_types.py +9 -1
  122. biolib/compute_node/job_worker/executors/types.py +19 -9
  123. biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
  124. biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
  125. biolib/compute_node/job_worker/job_storage.py +108 -0
  126. biolib/compute_node/job_worker/job_worker.py +397 -212
  127. biolib/compute_node/job_worker/large_file_system.py +87 -38
  128. biolib/compute_node/job_worker/network_alloc.py +99 -0
  129. biolib/compute_node/job_worker/network_buffer.py +240 -0
  130. biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
  131. biolib/compute_node/job_worker/utils.py +9 -24
  132. biolib/compute_node/remote_host_proxy.py +400 -98
  133. biolib/compute_node/utils.py +31 -9
  134. biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
  135. biolib/compute_node/webserver/proxy_utils.py +28 -0
  136. biolib/compute_node/webserver/webserver.py +130 -44
  137. biolib/compute_node/webserver/webserver_types.py +2 -6
  138. biolib/compute_node/webserver/webserver_utils.py +77 -12
  139. biolib/compute_node/webserver/worker_thread.py +183 -42
  140. biolib/experiments/__init__.py +0 -0
  141. biolib/experiments/experiment.py +356 -0
  142. biolib/jobs/__init__.py +1 -0
  143. biolib/jobs/job.py +741 -0
  144. biolib/jobs/job_result.py +185 -0
  145. biolib/jobs/types.py +50 -0
  146. biolib/py.typed +0 -0
  147. biolib/runtime/__init__.py +14 -0
  148. biolib/sdk/__init__.py +91 -0
  149. biolib/tables.py +34 -0
  150. biolib/typing_utils.py +2 -7
  151. biolib/user/__init__.py +1 -0
  152. biolib/user/sign_in.py +54 -0
  153. biolib/utils/__init__.py +162 -0
  154. biolib/utils/cache_state.py +94 -0
  155. biolib/utils/multipart_uploader.py +194 -0
  156. biolib/utils/seq_util.py +150 -0
  157. biolib/utils/zip/remote_zip.py +640 -0
  158. pybiolib-1.2.1890.dist-info/METADATA +41 -0
  159. pybiolib-1.2.1890.dist-info/RECORD +177 -0
  160. {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
  161. pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
  162. README.md +0 -17
  163. biolib/app/app_result.py +0 -68
  164. biolib/app/utils.py +0 -62
  165. biolib/biolib-js/0-biolib.worker.js +0 -1
  166. biolib/biolib-js/1-biolib.worker.js +0 -1
  167. biolib/biolib-js/2-biolib.worker.js +0 -1
  168. biolib/biolib-js/3-biolib.worker.js +0 -1
  169. biolib/biolib-js/4-biolib.worker.js +0 -1
  170. biolib/biolib-js/5-biolib.worker.js +0 -1
  171. biolib/biolib-js/6-biolib.worker.js +0 -1
  172. biolib/biolib-js/index.html +0 -10
  173. biolib/biolib-js/main-biolib.js +0 -1
  174. biolib/biolib_api_client/biolib_account_api.py +0 -21
  175. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
  176. biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
  177. biolib/biolib_binary_format/module_output.py +0 -58
  178. biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
  179. biolib/biolib_push.py +0 -114
  180. biolib/cli.py +0 -203
  181. biolib/cli_utils.py +0 -273
  182. biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
  183. biolib/compute_node/enclave/__init__.py +0 -2
  184. biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
  185. biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
  186. biolib/compute_node/job_worker/executors/base_executor.py +0 -18
  187. biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
  188. biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
  189. biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
  190. biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
  191. biolib/lfs.py +0 -196
  192. biolib/pyppeteer/.circleci/config.yml +0 -100
  193. biolib/pyppeteer/.coveragerc +0 -3
  194. biolib/pyppeteer/.gitignore +0 -89
  195. biolib/pyppeteer/.pre-commit-config.yaml +0 -28
  196. biolib/pyppeteer/CHANGES.md +0 -253
  197. biolib/pyppeteer/CONTRIBUTING.md +0 -26
  198. biolib/pyppeteer/LICENSE +0 -12
  199. biolib/pyppeteer/README.md +0 -137
  200. biolib/pyppeteer/docs/Makefile +0 -177
  201. biolib/pyppeteer/docs/_static/custom.css +0 -28
  202. biolib/pyppeteer/docs/_templates/layout.html +0 -10
  203. biolib/pyppeteer/docs/changes.md +0 -1
  204. biolib/pyppeteer/docs/conf.py +0 -299
  205. biolib/pyppeteer/docs/index.md +0 -21
  206. biolib/pyppeteer/docs/make.bat +0 -242
  207. biolib/pyppeteer/docs/reference.md +0 -211
  208. biolib/pyppeteer/docs/server.py +0 -60
  209. biolib/pyppeteer/poetry.lock +0 -1699
  210. biolib/pyppeteer/pyppeteer/__init__.py +0 -135
  211. biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
  212. biolib/pyppeteer/pyppeteer/browser.py +0 -401
  213. biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
  214. biolib/pyppeteer/pyppeteer/command.py +0 -22
  215. biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
  216. biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
  217. biolib/pyppeteer/pyppeteer/coverage.py +0 -346
  218. biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
  219. biolib/pyppeteer/pyppeteer/dialog.py +0 -79
  220. biolib/pyppeteer/pyppeteer/domworld.py +0 -597
  221. biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
  222. biolib/pyppeteer/pyppeteer/errors.py +0 -48
  223. biolib/pyppeteer/pyppeteer/events.py +0 -63
  224. biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
  225. biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
  226. biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
  227. biolib/pyppeteer/pyppeteer/helpers.py +0 -245
  228. biolib/pyppeteer/pyppeteer/input.py +0 -371
  229. biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
  230. biolib/pyppeteer/pyppeteer/launcher.py +0 -683
  231. biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
  232. biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
  233. biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
  234. biolib/pyppeteer/pyppeteer/multimap.py +0 -82
  235. biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
  236. biolib/pyppeteer/pyppeteer/options.py +0 -8
  237. biolib/pyppeteer/pyppeteer/page.py +0 -1728
  238. biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
  239. biolib/pyppeteer/pyppeteer/target.py +0 -147
  240. biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
  241. biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
  242. biolib/pyppeteer/pyppeteer/tracing.py +0 -93
  243. biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
  244. biolib/pyppeteer/pyppeteer/util.py +0 -18
  245. biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
  246. biolib/pyppeteer/pyppeteer/worker.py +0 -101
  247. biolib/pyppeteer/pyproject.toml +0 -97
  248. biolib/pyppeteer/spell.txt +0 -137
  249. biolib/pyppeteer/tox.ini +0 -72
  250. biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
  251. biolib/start_cli.py +0 -7
  252. biolib/utils.py +0 -47
  253. biolib/validators/validate_app_version.py +0 -183
  254. biolib/validators/validate_argument.py +0 -134
  255. biolib/validators/validate_module.py +0 -323
  256. biolib/validators/validate_zip_file.py +0 -40
  257. biolib/validators/validator_utils.py +0 -103
  258. pybiolib-0.2.951.dist-info/LICENSE +0 -21
  259. pybiolib-0.2.951.dist-info/METADATA +0 -61
  260. pybiolib-0.2.951.dist-info/RECORD +0 -153
  261. pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
  262. /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
biolib/cli_utils.py DELETED
@@ -1,273 +0,0 @@
1
- import os
2
- import shutil
3
- import sys
4
- import platform
5
- import tempfile
6
- import time
7
- from pathlib import Path
8
- from typing import Tuple
9
- from zipfile import ZipFile
10
- from termcolor import colored
11
-
12
- import yaml
13
-
14
- from biolib.biolib_api_client import AppVersionOnJob, Module, FilesMapping
15
- from biolib.validators.validate_app_version import validate_app_version # type: ignore
16
- from biolib.validators.validate_argument import validate_argument # type: ignore
17
- from biolib.validators.validate_module import validate_module # type: ignore
18
- from biolib.biolib_errors import BioLibError
19
- from biolib.biolib_logging import logger
20
- from biolib.typing_utils import Dict, List
21
-
22
-
23
- class BiolibValidationError(Exception):
24
- pass
25
-
26
-
27
- def bl_colored(string, colour):
28
- # The colors does not seem to play nice with Windows PowerShell terminal
29
- if platform.system() == 'Windows':
30
- return string
31
- else:
32
- return colored(string, colour)
33
-
34
-
35
- def validate_and_get_args(args):
36
- if args is None:
37
- args = []
38
-
39
- if isinstance(args, str):
40
- args = list(filter(lambda p: p != '', args.split(' ')))
41
-
42
- if not isinstance(args, list):
43
- raise Exception('The given input arguments must be list or str')
44
-
45
- return args
46
-
47
-
48
- def get_files_dict_and_file_args(files, args):
49
- if files is None:
50
- files = []
51
- # TODO: Figure out how to make this slightly less error prone
52
- for idx, arg in enumerate(args):
53
- if not arg.startswith('/'):
54
- arg = f'{os.getcwd()}/{arg}'
55
-
56
- arg = Path(arg)
57
- if arg.is_file():
58
- files.append(arg)
59
- # Make sure that arg is only the filename
60
- args[idx] = arg.parts[-1]
61
-
62
- files_dict = {}
63
- for file_path in files:
64
- file = open(file_path, 'rb')
65
- path = '/' + file_path.parts[-1]
66
-
67
- files_dict[path] = file.read()
68
- file.close()
69
-
70
- return files_dict, args
71
-
72
-
73
- def validate_and_get_app_path(provided_app_path):
74
- if provided_app_path.startswith('/'):
75
- app_path = Path(provided_app_path)
76
- else:
77
- app_path = Path(os.getcwd()) / provided_app_path
78
-
79
- if not app_path.exists():
80
- print(f'ERROR: Could not find path {app_path}')
81
- sys.exit(1)
82
-
83
- if not app_path.is_dir():
84
- print(f'ERROR: Path {app_path} is not a directory')
85
- sys.exit(1)
86
-
87
- return app_path
88
-
89
-
90
- def get_yaml_data(app_path):
91
- try:
92
- yaml_file = open(f'{app_path}/.biolib/config.yml', 'r', encoding='utf-8')
93
-
94
- except Exception as error: # pylint: disable=broad-except
95
- raise BioLibError(f'Could not open the biolib config file at {app_path}/.biolib/config.yml') from error
96
-
97
- try:
98
- yaml_data = yaml.safe_load(yaml_file)
99
-
100
- except Exception as error: # pylint: disable=broad-except
101
- raise BioLibError(f'Could not parse {app_path}/.biolib/config.yml. Please make sure it is valid YAML') \
102
- from error
103
-
104
- return yaml_data
105
-
106
-
107
- def validate_arguments(yaml_data):
108
- error_dict = {'arguments': {}}
109
- # Arguments are optional
110
- if 'arguments' in yaml_data:
111
- for argument in yaml_data['arguments']:
112
- argument_errors = validate_argument(argument)
113
- if argument_errors:
114
- error_dict['arguments'].update(argument_errors)
115
-
116
- # Just return empty dict if we have no errors
117
- if error_dict['arguments']:
118
- return error_dict
119
- else:
120
- return {}
121
-
122
-
123
- def validate_and_get_biolib_yaml_version(yaml_data, error_dict):
124
- if not 'biolib_version' in yaml_data.keys():
125
- error_dict['config_yml']['biolib_version'] = ['Your config file is missing the biolib_version field.']
126
- print_errors(error_dict)
127
- sys.exit(1)
128
- else:
129
- biolib_version = yaml_data['biolib_version']
130
-
131
- if biolib_version not in (1, 2):
132
- error_dict['config_yml']['biolib_version'] = \
133
- [f'Biolib version can only be either 1 or 2 for now, your version is {biolib_version}.']
134
- print_errors(error_dict)
135
- sys.exit(1)
136
-
137
- return biolib_version
138
-
139
-
140
- def validate_modules(yaml_data, biolib_yaml_version):
141
- error_dict = {'modules': {}}
142
- if 'modules' in yaml_data:
143
- for name, task_data in yaml_data['modules'].items():
144
- task_errors = validate_module(name, task_data, biolib_yaml_version)
145
- if task_errors:
146
- error_dict['modules'].update(task_errors)
147
-
148
- # Just return empty dict if we have no errors
149
- if error_dict['modules']:
150
- return error_dict
151
- else:
152
- return {}
153
-
154
-
155
- def validate_yaml_config(yaml_data, source_files_zip_path):
156
- source_files_zip = ZipFile(source_files_zip_path)
157
- error_dict = {'config_yml': {}}
158
-
159
- biolib_yaml_version = validate_and_get_biolib_yaml_version(yaml_data, error_dict)
160
- error_dict['config_yml'].update(validate_app_version(yaml_data, biolib_yaml_version, source_files_zip))
161
- error_dict['config_yml'].update(validate_modules(yaml_data, biolib_yaml_version))
162
- error_dict['config_yml'].update(validate_arguments(yaml_data))
163
-
164
- if error_dict['config_yml']:
165
- print_errors(error_dict)
166
- raise BiolibValidationError('Could not validate .biolib/config.yml')
167
-
168
-
169
- def print_errors(error_dict):
170
- print(bl_colored("\nThe following validation errors were found in the .biolib/config.yml file:", 'red'))
171
- print(yaml.safe_dump(error_dict, allow_unicode=True, default_flow_style=False))
172
-
173
-
174
- def write_output_files(mapped_output_files=None):
175
- output_dir_path = 'biolib_results'
176
-
177
- if os.path.exists(output_dir_path):
178
- os.rename(output_dir_path, f'{output_dir_path}_old_{time.strftime("%Y%m%d%H%M%S")}')
179
-
180
- if mapped_output_files:
181
- logger.debug("Output Files:")
182
-
183
- for path, data in mapped_output_files.items():
184
- dirs, file = os.path.split(path)
185
- path_on_disk = f'{output_dir_path}{path}'
186
-
187
- if dirs:
188
- os.makedirs(f'{output_dir_path}{dirs}', exist_ok=True)
189
- if file:
190
- open(path_on_disk, 'wb').write(data)
191
-
192
- logger.debug(f" - {path_on_disk}")
193
-
194
- logger.debug('\n')
195
-
196
-
197
- def get_pretty_print_module_output_string(stdout=None, stderr=None, exitcode=None) -> str:
198
- stderr_string = f"\n{bl_colored('stderr:', 'red')}\n{stderr.decode()}" if stderr else ''
199
- return f'''{bl_colored('stdout:', 'green')}
200
- {stdout.decode()}{stderr_string}
201
- {bl_colored('exitcode:', 'green' if exitcode == 0 else 'red')} {exitcode}'''
202
-
203
-
204
- def _get_files_mappings(file_commands: List[str]) -> List[FilesMapping]:
205
- files_mappings: List[FilesMapping] = []
206
- for file_command in file_commands:
207
- # TODO: figure out if splitting on space is good enough
208
- _, from_path, to_path = file_command.split(' ')
209
- files_mappings.append(FilesMapping(from_path=from_path, to_path=to_path))
210
- return files_mappings
211
-
212
-
213
- def get_mocked_app_version_from_yaml(yaml_data: Dict, source_files_zip_path: str):
214
- modules: List[Module] = []
215
- for module_name in yaml_data['modules']:
216
- yaml_module = yaml_data['modules'][module_name]
217
-
218
- image_hostname, image_uri = yaml_module['image'].split('://')
219
- if image_hostname != 'local-docker':
220
- raise Exception(
221
- f'Found image {image_uri} with hostname {image_hostname}, please use "local-docker://"'
222
- f'Your application must use modules with local docker images to use run-dev'
223
- )
224
-
225
- modules.append(Module(
226
- command=yaml_module.get('command', ''),
227
- environment='biolib-ecr',
228
- image_uri=image_uri,
229
- input_files_mappings=_get_files_mappings(yaml_module['input_files']),
230
- large_file_systems=[],
231
- name=module_name,
232
- output_files_mappings=_get_files_mappings(yaml_module['output_files']),
233
- source_files_mappings=_get_files_mappings(yaml_module['source_files']),
234
- working_directory=yaml_module['working_directory'],
235
- estimated_image_size_bytes=None # This should not be provided here
236
- ))
237
-
238
- return AppVersionOnJob(
239
- client_side_executable_zip=source_files_zip_path,
240
- consumes_stdin=True,
241
- is_runnable_by_user=True,
242
- modules=modules,
243
- public_id='app-version-mock-id',
244
- remote_hosts=[],
245
- settings=[],
246
- stdout_render_type='text',
247
- )
248
-
249
-
250
- def create_temporary_directory_with_source_files_zip(app_path: str) -> Tuple[tempfile.TemporaryDirectory, str]:
251
- original_working_directory = os.getcwd()
252
- temporary_directory = tempfile.TemporaryDirectory()
253
- os.chdir(app_path)
254
- try:
255
- zip_path_without_file_extension = os.path.join(temporary_directory.name, 'biolib-cli-tmp-source-files-zip')
256
- zip_path = f'{zip_path_without_file_extension}.zip'
257
- app_folder_name = Path(app_path).absolute().parts[-1]
258
- shutil.make_archive(
259
- base_name=zip_path_without_file_extension,
260
- format='zip',
261
- root_dir='..',
262
- base_dir=app_folder_name,
263
- )
264
-
265
- return temporary_directory, zip_path
266
-
267
- except Exception as error:
268
- temporary_directory.cleanup()
269
- raise Exception('Failed to create zip of application') from error
270
-
271
- finally:
272
- # change back to old working directory
273
- os.chdir(original_working_directory)
@@ -1,7 +0,0 @@
1
- from biolib.typing_utils import TypedDict
2
-
3
-
4
- class VsockProxyResponse(TypedDict):
5
- hostname: str
6
- id: str
7
- port: int
@@ -1,2 +0,0 @@
1
- # The CID for connections from the enclave to the parent
2
- PARENT_CID = 3
@@ -1,53 +0,0 @@
1
- from urllib.parse import urlparse
2
- from biolib.biolib_api_client import RemoteHost
3
- from biolib.biolib_docker_client import BiolibDockerClient
4
- from biolib.biolib_logging import logger
5
- from biolib.compute_node.remote_host_proxy import RemoteHostProxy
6
- from biolib.compute_node.webserver.webserver_types import WebserverConfig
7
- from biolib.typing_utils import List
8
-
9
-
10
-
11
- def start_enclave_remote_hosts(config: WebserverConfig) -> None:
12
- logger.debug('Starting Docker network for enclave remote host proxies')
13
- docker = BiolibDockerClient.get_docker_client()
14
- public_network = docker.networks.create(
15
- driver='bridge',
16
- internal=False,
17
- name='biolib-enclave-remote-hosts-network',
18
- )
19
-
20
- biolib_remote_hosts: List[RemoteHost] = []
21
- biolib_remote_host_proxies: List[RemoteHostProxy] = []
22
- base_hostname = urlparse(config['base_url']).hostname
23
- # Make sure base_hostname is not None for typing reasons
24
- if not base_hostname:
25
- raise Exception('Base hostname not set, likely due to base url not being set. This is required in enclaves')
26
-
27
- if base_hostname == 'biolib.com':
28
- biolib_remote_hosts.append(RemoteHost(hostname='containers.biolib.com'))
29
- elif base_hostname == 'staging-elb.biolib.com':
30
- biolib_remote_hosts.append(RemoteHost(hostname='containers.staging.biolib.com'))
31
-
32
- # Allow reaching backend for self registering and job creation
33
- biolib_remote_hosts.append(RemoteHost(hostname=base_hostname))
34
- # For downloading container image layers from S3 URLs returned by ECR proxy
35
- biolib_remote_hosts.append(RemoteHost(
36
- hostname=f"prod-{config['ecr_region_name']}-starport-layer-bucket.s3.{config['ecr_region_name']}.amazonaws.com"
37
- ))
38
- # For downloading source files zip
39
- biolib_remote_hosts.append(RemoteHost(
40
- hostname=f"{config['s3_general_storage_bucket_name']}.s3.amazonaws.com"
41
- ))
42
-
43
- logger.debug('Starting enclave remote host proxies')
44
- for remote_host in biolib_remote_hosts:
45
- remote_host_proxy = RemoteHostProxy(remote_host, public_network, internal_network=None, job_id=None)
46
- remote_host_proxy.start()
47
- biolib_remote_host_proxies.append(remote_host_proxy)
48
-
49
- logger.debug('Writing to enclave /etc/hosts')
50
- with open('/etc/hosts', mode='a') as hosts_file:
51
- for proxy in biolib_remote_host_proxies:
52
- ip_address = proxy.get_ip_address_on_network(public_network)
53
- hosts_file.write(f'\n{ip_address} {proxy.hostname}')
@@ -1,64 +0,0 @@
1
- """
2
- This file is modified based on donkersgoed's repository (https://github.com/donkersgoed/nitropepper-enclave-app)
3
- """
4
-
5
- import Crypto
6
- from Crypto.PublicKey import RSA
7
- from Crypto.Cipher import PKCS1_OAEP
8
- from Crypto.Hash import SHA256
9
-
10
- # enclave_build will only be present when running on an enclave
11
- import biolib.compute_node.enclave.libnsm as libnsm # type: ignore # pylint: disable=import-error, no-name-in-module
12
-
13
-
14
- class NitroSecureModuleUtils:
15
- """NSM util class."""
16
-
17
- def __init__(self):
18
- """Construct a new NSMUtil instance."""
19
- # Initialize the Rust NSM Library
20
- self._nsm_fd = libnsm.nsm_lib_init() # pylint:disable=c-extension-no-member
21
- # Create a new random function `nsm_rand_func`, which
22
- # utilizes the NSM module.
23
- self.nsm_rand_func = lambda num_bytes: libnsm.nsm_get_random( # pylint:disable=c-extension-no-member
24
- self._nsm_fd, num_bytes
25
- )
26
-
27
- # Force pycryptodome to use the new rand function.
28
- # Without this, pycryptodome defaults to /dev/random
29
- # and /dev/urandom, which are not available in Enclaves.
30
- self._monkey_patch_crypto(self.nsm_rand_func)
31
-
32
- # Generate a new RSA certificate, which will be used to
33
- # generate the Attestation document and to decrypt results
34
- # for KMS Decrypt calls with this document.
35
- self._rsa_key = RSA.generate(2048)
36
- self._public_key = self._rsa_key.publickey().export_key('DER')
37
-
38
- def get_attestation_doc(self):
39
- """Get the attestation document from /dev/nsm."""
40
- libnsm_att_doc_cose_signed = libnsm.nsm_get_attestation_doc( # pylint:disable=c-extension-no-member
41
- self._nsm_fd,
42
- self._public_key,
43
- len(self._public_key)
44
- )
45
- return libnsm_att_doc_cose_signed
46
-
47
- def decrypt(self, ciphertext):
48
- """
49
- Decrypt ciphertext using private key
50
- """
51
- cipher = PKCS1_OAEP.new(self._rsa_key, hashAlgo=SHA256)
52
- plaintext = cipher.decrypt(ciphertext)
53
-
54
- return plaintext
55
-
56
- @classmethod
57
- def _monkey_patch_crypto(cls, nsm_rand_func):
58
- """Monkeypatch Crypto to use the NSM rand function."""
59
- Crypto.Random.get_random_bytes = nsm_rand_func
60
-
61
- def new_random_read(self, n_bytes): # pylint:disable=unused-argument
62
- return nsm_rand_func(n_bytes)
63
-
64
- Crypto.Random._UrandomRNG.read = new_random_read # pylint:disable=protected-access
@@ -1,18 +0,0 @@
1
- import abc
2
-
3
- from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions
4
-
5
-
6
- class BaseExecutor(abc.ABC):
7
-
8
- def __init__(self, options: LocalExecutorOptions):
9
- self._options: LocalExecutorOptions = options
10
- self._is_cleaning_up = False
11
-
12
- @abc.abstractmethod
13
- def execute_module(self, module_input_serialized: bytes) -> bytes:
14
- pass
15
-
16
- @abc.abstractmethod
17
- def cleanup(self) -> None:
18
- pass
@@ -1,173 +0,0 @@
1
- import http.server
2
- import json
3
- import socketserver
4
- import threading
5
- import os
6
- import asyncio
7
- import subprocess
8
- import base64
9
- import sys
10
- import logging
11
-
12
- from biolib.typing_utils import Optional
13
- from biolib import utils
14
- from biolib.biolib_api_client import BiolibApiClient
15
- from biolib.biolib_errors import BioLibError
16
- from biolib.biolib_logging import logger, TRACE
17
- from biolib.compute_node.job_worker.executors.base_executor import BaseExecutor
18
- from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions
19
- from biolib.pyppeteer.pyppeteer import launch, command # type: ignore
20
- from biolib.pyppeteer.pyppeteer.launcher import resolveExecutablePath, __chromium_revision__ # type: ignore
21
-
22
- # specifically limit logs from pyppeteer
23
- logging.getLogger('biolib.pyppeteer.pyppeteer').setLevel(logging.ERROR)
24
- logging.getLogger('biolib.pyppeteer.pyppeteer.connection').setLevel(logging.ERROR)
25
-
26
-
27
- class RequestHandler(http.server.SimpleHTTPRequestHandler):
28
- def log_message(self, format, *args): # pylint: disable=redefined-builtin
29
- if logger.level == TRACE:
30
- http.server.SimpleHTTPRequestHandler.log_message(self, format, *args)
31
-
32
-
33
- class PyppeteerExecutor(BaseExecutor):
34
- _chrome_executable_path: Optional[str] = None
35
- _no_sandbox: bool = True
36
-
37
- def execute_module(self, module_input_serialized: bytes) -> bytes:
38
- with socketserver.TCPServer(('127.0.0.1', 0), RequestHandler) as httpd:
39
- port = httpd.server_address[1]
40
- thread = threading.Thread(target=httpd.serve_forever)
41
- original_working_directory = os.getcwd()
42
- # TODO: figure out how we can avoid changing the current directory
43
- biolib_js_dir = os.path.join(utils.BIOLIB_PACKAGE_ROOT_DIR, 'biolib-js')
44
- os.chdir(biolib_js_dir)
45
- try:
46
- thread.start()
47
- module_output_serialized: bytes = asyncio.get_event_loop().run_until_complete(self._call_pyppeteer(
48
- port,
49
- self._options,
50
- module_input_serialized,
51
- ))
52
- return module_output_serialized
53
- finally:
54
- os.chdir(original_working_directory)
55
- httpd.shutdown()
56
- thread.join()
57
-
58
- async def _call_pyppeteer(self, port: int, options: LocalExecutorOptions, module_input_serialized: bytes):
59
- if not self._chrome_executable_path:
60
- mac_chrome_path = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
61
- if os.path.isfile(mac_chrome_path):
62
- self._chrome_executable_path = mac_chrome_path
63
-
64
- if not self._chrome_executable_path:
65
- linux_chrome_path = '/usr/lib/chromium-browser/chromium-browser'
66
-
67
- # special install for google colab
68
- if not os.path.isfile(linux_chrome_path) and 'google.colab' in sys.modules:
69
- subprocess.run('apt-get update', shell=True, check=True)
70
- subprocess.run('apt install chromium-chromedriver', shell=True, check=True)
71
-
72
- if os.path.isfile(linux_chrome_path):
73
- self._chrome_executable_path = linux_chrome_path
74
-
75
- resolved_path = resolveExecutablePath(None, __chromium_revision__)
76
- if not self._chrome_executable_path and resolved_path[1]:
77
- # if executable_path is not set explicit,
78
- # and resolveExecutablePath failed (== we got an error message back in resolved_path[1])
79
- logger.info('Installing dependencies...')
80
- os.environ['PYPPETEER_NO_PROGRESS_BAR'] = 'true'
81
- command.install()
82
-
83
- logger.info('Computing...')
84
-
85
- chrome_arguments = [
86
- '--disable-web-security',
87
- ]
88
- if self._no_sandbox:
89
- chrome_arguments.append('--no-sandbox')
90
-
91
- browser = await launch(args=chrome_arguments, executablePath=self._chrome_executable_path)
92
-
93
- # start new page
94
- page = await browser.newPage()
95
-
96
- await page.goto('http://localhost:' + str(port))
97
-
98
- def get_data():
99
- return base64.b64encode(module_input_serialized).decode('ascii')
100
-
101
- def set_progress_compute(value):
102
- logger.debug(f'Compute progress: {value}')
103
-
104
- def set_progress_initialization(value):
105
- logger.debug(f'Initialization progress: {value}')
106
-
107
- def add_log_message(value):
108
- logger.debug(f'Log message: {value}')
109
-
110
- await page.exposeFunction('getData', get_data)
111
- await page.exposeFunction('setProgressCompute', set_progress_compute)
112
- await page.exposeFunction('setProgressInitialization', set_progress_initialization)
113
- await page.exposeFunction('addLogMessage', add_log_message)
114
-
115
- api_client = BiolibApiClient.get()
116
- refresh_token_js_value = 'undefined' if not api_client.refresh_token else f'"{api_client.refresh_token}"'
117
-
118
- job = options['job'].copy()
119
- if 'custom_compute_node_url' in job:
120
- job.pop('custom_compute_node_url')
121
-
122
- job_json_string = json.dumps(job)
123
-
124
- output_serialized_js_bytes = await page.evaluate('''
125
- async function() {
126
- const refreshToken = ''' + refresh_token_js_value + ''';
127
- const baseUrl = \'''' + options['biolib_base_url'] + '''\';
128
- const job = ''' + job_json_string + ''';
129
-
130
- const { BioLibSingleton, AppClient } = window.BioLib;
131
- BioLibSingleton.setConfig({ baseUrl, refreshToken });
132
- AppClient.setApiClient(BioLibSingleton.get());
133
-
134
- const inputBase64 = await window.getData();
135
- const inputByteArray = Uint8Array.from(atob(inputBase64), c => c.charCodeAt(0));
136
-
137
- const jobUtils = {
138
- setProgressCompute: window.setProgressCompute,
139
- setProgressInitialization: window.setProgressInitialization,
140
- addLogMessage: window.addLogMessage,
141
- };
142
-
143
- try {
144
- const moduleOutput = await AppClient.runJob(job, inputByteArray, jobUtils);
145
- return moduleOutput.serialize();
146
- } catch(err) {
147
- return err.toString();
148
- }
149
- }
150
- ''')
151
- logger.debug('Closing browser')
152
- await browser.close()
153
-
154
- module_output_serialized = self._js_to_python_byte_array_converter(output_serialized_js_bytes)
155
-
156
- # TODO: check if module_output is correctly serialized
157
- if isinstance(module_output_serialized, bytes):
158
- return module_output_serialized
159
- else:
160
- raise BioLibError(module_output_serialized)
161
-
162
- def cleanup(self):
163
- pass
164
-
165
- @staticmethod
166
- def _js_to_python_byte_array_converter(js_encoded) -> bytes:
167
- try:
168
- return bytes(list([js_encoded[str(i)] for i in range(len(js_encoded))]))
169
- except Exception as error:
170
- logger.error('Failed to decode response from browser')
171
- logger.error(error)
172
- logger.error(js_encoded)
173
- raise BioLibError(js_encoded) from error
@@ -1 +0,0 @@
1
- from biolib.compute_node.job_worker.executors.remote.remote_executor import RemoteExecutor
@@ -1,81 +0,0 @@
1
- import cbor2 # type: ignore
2
- import cose # type: ignore
3
-
4
- from cose import EC2, CoseAlgorithms, CoseEllipticCurves
5
- from Crypto.Util.number import long_to_bytes
6
- from OpenSSL import crypto
7
-
8
- from biolib import utils
9
- from biolib.biolib_errors import BioLibError
10
-
11
-
12
- class NitroEnclaveUtils:
13
- def attest_enclave_and_get_rsa_public_key(self, expected_pcrs_and_aws_cert, attestation_document_bytes):
14
- cbor_data = cbor2.loads(attestation_document_bytes)
15
- cbor_document = cbor2.loads(cbor_data[2])
16
- expected_pcrs_list = expected_pcrs_and_aws_cert['pcrObjectArray']
17
- aws_nitro_root_cert_pem = "-----BEGIN CERTIFICATE-----\n" + \
18
- expected_pcrs_and_aws_cert['awsNitroRootCertificateBase64'] + \
19
- "\n-----END CERTIFICATE-----"
20
- actual_pcrs = {str(k): v.hex() for k, v in cbor_document['pcrs'].items()}
21
-
22
- if not utils.BIOLIB_CLOUD_SKIP_PCR_VERIFICATION:
23
- self._assert_pcr_validity(actual_pcrs, expected_pcrs_list)
24
-
25
- cert = crypto.load_certificate(crypto.FILETYPE_ASN1, cbor_document['certificate'])
26
- self._assert_certificate_chain_validity(cbor_document['cabundle'], cert, aws_nitro_root_cert_pem)
27
- self._assert_cose_signature_validity(cert, cbor_data)
28
- return cbor_document['public_key']
29
-
30
- def _assert_pcr_validity(self, actual_pcrs, expected_pcrs_list):
31
- for expected_pcrs in expected_pcrs_list:
32
- if self._does_pcr_object_match_actual(actual_pcrs, expected_pcrs):
33
- return True
34
-
35
- # Raise error if no match was fund
36
- raise BioLibError('Failed to verify PCRs')
37
-
38
- def _does_pcr_object_match_actual(self, actual_pcrs, expected_pcrs):
39
- for pcr_key in expected_pcrs.keys():
40
- if actual_pcrs[pcr_key] != expected_pcrs[pcr_key]:
41
- return False
42
- # If all matched return True
43
- return True
44
-
45
- def _assert_certificate_chain_validity(self, cabundle, cert, aws_nitro_root_cert_pem):
46
- store = crypto.X509Store()
47
- _cert = crypto.load_certificate(crypto.FILETYPE_PEM, aws_nitro_root_cert_pem)
48
- store.add_cert(_cert)
49
-
50
- for _cert_binary in cabundle[1:]:
51
- _cert = crypto.load_certificate(crypto.FILETYPE_ASN1, _cert_binary)
52
- store.add_cert(_cert)
53
-
54
- store_ctx = crypto.X509StoreContext(store, cert)
55
- try:
56
- store_ctx.verify_certificate()
57
- except Exception as exception:
58
- raise BioLibError('Failed to verify certificates') from exception
59
-
60
- def _assert_cose_signature_validity(self, cert, cbor_data):
61
- phdr = cbor2.loads(cbor_data[0])
62
- uhdr = cbor_data[1]
63
- signature = cbor_data[3]
64
-
65
- cert_public_numbers = cert.get_pubkey().to_cryptography_key().public_numbers()
66
- x_long = cert_public_numbers.x
67
- y_long = cert_public_numbers.y
68
-
69
- x_bytes = long_to_bytes(x_long)
70
- y_bytes = long_to_bytes(y_long)
71
-
72
- # Create the EC2 key from public key parameters
73
- key = EC2(alg=CoseAlgorithms.ES384, x=x_bytes, y=y_bytes, crv=CoseEllipticCurves.P_384)
74
-
75
- # Construct the Sign1 message
76
- msg = cose.Sign1Message(phdr=phdr, uhdr=uhdr, payload=cbor_data[2])
77
- msg.signature = signature
78
-
79
- # Verify the signature using the EC2 key
80
- if not msg.verify_signature(key):
81
- raise Exception('Failed to verify signature in attestation document')