pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/__init__.py +357 -11
- biolib/_data_record/data_record.py +380 -0
- biolib/_index/__init__.py +0 -0
- biolib/_index/index.py +55 -0
- biolib/_index/query_result.py +103 -0
- biolib/_internal/__init__.py +0 -0
- biolib/_internal/add_copilot_prompts.py +58 -0
- biolib/_internal/add_gui_files.py +81 -0
- biolib/_internal/data_record/__init__.py +1 -0
- biolib/_internal/data_record/data_record.py +85 -0
- biolib/_internal/data_record/push_data.py +116 -0
- biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
- biolib/_internal/errors.py +5 -0
- biolib/_internal/file_utils.py +125 -0
- biolib/_internal/fuse_mount/__init__.py +1 -0
- biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
- biolib/_internal/http_client.py +159 -0
- biolib/_internal/lfs/__init__.py +1 -0
- biolib/_internal/lfs/cache.py +51 -0
- biolib/_internal/libs/__init__.py +1 -0
- biolib/_internal/libs/fusepy/__init__.py +1257 -0
- biolib/_internal/push_application.py +488 -0
- biolib/_internal/runtime.py +22 -0
- biolib/_internal/string_utils.py +13 -0
- biolib/_internal/templates/__init__.py +1 -0
- biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
- biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
- biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
- biolib/_internal/templates/gitignore_template/.gitignore +10 -0
- biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
- biolib/_internal/templates/gui_template/App.tsx +53 -0
- biolib/_internal/templates/gui_template/Dockerfile +27 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/index.css +5 -0
- biolib/_internal/templates/gui_template/index.html +13 -0
- biolib/_internal/templates/gui_template/index.tsx +10 -0
- biolib/_internal/templates/gui_template/package.json +27 -0
- biolib/_internal/templates/gui_template/tsconfig.json +24 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
- biolib/_internal/templates/gui_template/vite.config.mts +10 -0
- biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
- biolib/_internal/templates/init_template/Dockerfile +14 -0
- biolib/_internal/templates/init_template/requirements.txt +1 -0
- biolib/_internal/templates/init_template/run.py +12 -0
- biolib/_internal/templates/init_template/run.sh +4 -0
- biolib/_internal/templates/templates.py +25 -0
- biolib/_internal/tree_utils.py +106 -0
- biolib/_internal/utils/__init__.py +65 -0
- biolib/_internal/utils/auth.py +46 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_internal/utils/multinode.py +263 -0
- biolib/_runtime/runtime.py +157 -0
- biolib/_session/session.py +44 -0
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +74 -0
- biolib/_shared/types/account.py +12 -0
- biolib/_shared/types/account_member.py +8 -0
- biolib/_shared/types/app.py +9 -0
- biolib/_shared/types/data_record.py +40 -0
- biolib/_shared/types/experiment.py +32 -0
- biolib/_shared/types/file_node.py +17 -0
- biolib/_shared/types/push.py +6 -0
- biolib/_shared/types/resource.py +37 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/_shared/types/resource_permission.py +14 -0
- biolib/_shared/types/resource_version.py +19 -0
- biolib/_shared/types/result.py +14 -0
- biolib/_shared/types/typing.py +10 -0
- biolib/_shared/types/user.py +19 -0
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/__init__.py +6 -0
- biolib/api/client.py +168 -0
- biolib/app/app.py +252 -49
- biolib/app/search_apps.py +45 -0
- biolib/biolib_api_client/api_client.py +126 -31
- biolib/biolib_api_client/app_types.py +24 -4
- biolib/biolib_api_client/auth.py +31 -8
- biolib/biolib_api_client/biolib_app_api.py +147 -52
- biolib/biolib_api_client/biolib_job_api.py +161 -141
- biolib/biolib_api_client/job_types.py +21 -5
- biolib/biolib_api_client/lfs_types.py +7 -23
- biolib/biolib_api_client/user_state.py +56 -0
- biolib/biolib_binary_format/__init__.py +1 -4
- biolib/biolib_binary_format/file_in_container.py +105 -0
- biolib/biolib_binary_format/module_input.py +24 -7
- biolib/biolib_binary_format/module_output_v2.py +149 -0
- biolib/biolib_binary_format/remote_endpoints.py +34 -0
- biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
- biolib/biolib_binary_format/saved_job.py +3 -2
- biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
- biolib/biolib_binary_format/system_status_update.py +3 -2
- biolib/biolib_binary_format/utils.py +175 -0
- biolib/biolib_docker_client/__init__.py +11 -2
- biolib/biolib_errors.py +36 -0
- biolib/biolib_logging.py +27 -10
- biolib/cli/__init__.py +38 -0
- biolib/cli/auth.py +46 -0
- biolib/cli/data_record.py +164 -0
- biolib/cli/index.py +32 -0
- biolib/cli/init.py +421 -0
- biolib/cli/lfs.py +101 -0
- biolib/cli/push.py +50 -0
- biolib/cli/run.py +63 -0
- biolib/cli/runtime.py +14 -0
- biolib/cli/sdk.py +16 -0
- biolib/cli/start.py +56 -0
- biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
- biolib/compute_node/job_worker/cache_state.py +66 -88
- biolib/compute_node/job_worker/cache_types.py +1 -6
- biolib/compute_node/job_worker/docker_image_cache.py +112 -37
- biolib/compute_node/job_worker/executors/__init__.py +0 -3
- biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
- biolib/compute_node/job_worker/executors/docker_types.py +9 -1
- biolib/compute_node/job_worker/executors/types.py +19 -9
- biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
- biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
- biolib/compute_node/job_worker/job_storage.py +108 -0
- biolib/compute_node/job_worker/job_worker.py +397 -212
- biolib/compute_node/job_worker/large_file_system.py +87 -38
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
- biolib/compute_node/job_worker/utils.py +9 -24
- biolib/compute_node/remote_host_proxy.py +400 -98
- biolib/compute_node/utils.py +31 -9
- biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +130 -44
- biolib/compute_node/webserver/webserver_types.py +2 -6
- biolib/compute_node/webserver/webserver_utils.py +77 -12
- biolib/compute_node/webserver/worker_thread.py +183 -42
- biolib/experiments/__init__.py +0 -0
- biolib/experiments/experiment.py +356 -0
- biolib/jobs/__init__.py +1 -0
- biolib/jobs/job.py +741 -0
- biolib/jobs/job_result.py +185 -0
- biolib/jobs/types.py +50 -0
- biolib/py.typed +0 -0
- biolib/runtime/__init__.py +14 -0
- biolib/sdk/__init__.py +91 -0
- biolib/tables.py +34 -0
- biolib/typing_utils.py +2 -7
- biolib/user/__init__.py +1 -0
- biolib/user/sign_in.py +54 -0
- biolib/utils/__init__.py +162 -0
- biolib/utils/cache_state.py +94 -0
- biolib/utils/multipart_uploader.py +194 -0
- biolib/utils/seq_util.py +150 -0
- biolib/utils/zip/remote_zip.py +640 -0
- pybiolib-1.2.1890.dist-info/METADATA +41 -0
- pybiolib-1.2.1890.dist-info/RECORD +177 -0
- {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
- pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
- README.md +0 -17
- biolib/app/app_result.py +0 -68
- biolib/app/utils.py +0 -62
- biolib/biolib-js/0-biolib.worker.js +0 -1
- biolib/biolib-js/1-biolib.worker.js +0 -1
- biolib/biolib-js/2-biolib.worker.js +0 -1
- biolib/biolib-js/3-biolib.worker.js +0 -1
- biolib/biolib-js/4-biolib.worker.js +0 -1
- biolib/biolib-js/5-biolib.worker.js +0 -1
- biolib/biolib-js/6-biolib.worker.js +0 -1
- biolib/biolib-js/index.html +0 -10
- biolib/biolib-js/main-biolib.js +0 -1
- biolib/biolib_api_client/biolib_account_api.py +0 -21
- biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
- biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
- biolib/biolib_binary_format/module_output.py +0 -58
- biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
- biolib/biolib_push.py +0 -114
- biolib/cli.py +0 -203
- biolib/cli_utils.py +0 -273
- biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
- biolib/compute_node/enclave/__init__.py +0 -2
- biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
- biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
- biolib/compute_node/job_worker/executors/base_executor.py +0 -18
- biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
- biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
- biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
- biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
- biolib/lfs.py +0 -196
- biolib/pyppeteer/.circleci/config.yml +0 -100
- biolib/pyppeteer/.coveragerc +0 -3
- biolib/pyppeteer/.gitignore +0 -89
- biolib/pyppeteer/.pre-commit-config.yaml +0 -28
- biolib/pyppeteer/CHANGES.md +0 -253
- biolib/pyppeteer/CONTRIBUTING.md +0 -26
- biolib/pyppeteer/LICENSE +0 -12
- biolib/pyppeteer/README.md +0 -137
- biolib/pyppeteer/docs/Makefile +0 -177
- biolib/pyppeteer/docs/_static/custom.css +0 -28
- biolib/pyppeteer/docs/_templates/layout.html +0 -10
- biolib/pyppeteer/docs/changes.md +0 -1
- biolib/pyppeteer/docs/conf.py +0 -299
- biolib/pyppeteer/docs/index.md +0 -21
- biolib/pyppeteer/docs/make.bat +0 -242
- biolib/pyppeteer/docs/reference.md +0 -211
- biolib/pyppeteer/docs/server.py +0 -60
- biolib/pyppeteer/poetry.lock +0 -1699
- biolib/pyppeteer/pyppeteer/__init__.py +0 -135
- biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
- biolib/pyppeteer/pyppeteer/browser.py +0 -401
- biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
- biolib/pyppeteer/pyppeteer/command.py +0 -22
- biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
- biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
- biolib/pyppeteer/pyppeteer/coverage.py +0 -346
- biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
- biolib/pyppeteer/pyppeteer/dialog.py +0 -79
- biolib/pyppeteer/pyppeteer/domworld.py +0 -597
- biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
- biolib/pyppeteer/pyppeteer/errors.py +0 -48
- biolib/pyppeteer/pyppeteer/events.py +0 -63
- biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
- biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
- biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
- biolib/pyppeteer/pyppeteer/helpers.py +0 -245
- biolib/pyppeteer/pyppeteer/input.py +0 -371
- biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
- biolib/pyppeteer/pyppeteer/launcher.py +0 -683
- biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
- biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
- biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
- biolib/pyppeteer/pyppeteer/multimap.py +0 -82
- biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
- biolib/pyppeteer/pyppeteer/options.py +0 -8
- biolib/pyppeteer/pyppeteer/page.py +0 -1728
- biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
- biolib/pyppeteer/pyppeteer/target.py +0 -147
- biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
- biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
- biolib/pyppeteer/pyppeteer/tracing.py +0 -93
- biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
- biolib/pyppeteer/pyppeteer/util.py +0 -18
- biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
- biolib/pyppeteer/pyppeteer/worker.py +0 -101
- biolib/pyppeteer/pyproject.toml +0 -97
- biolib/pyppeteer/spell.txt +0 -137
- biolib/pyppeteer/tox.ini +0 -72
- biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
- biolib/start_cli.py +0 -7
- biolib/utils.py +0 -47
- biolib/validators/validate_app_version.py +0 -183
- biolib/validators/validate_argument.py +0 -134
- biolib/validators/validate_module.py +0 -323
- biolib/validators/validate_zip_file.py +0 -40
- biolib/validators/validator_utils.py +0 -103
- pybiolib-0.2.951.dist-info/LICENSE +0 -21
- pybiolib-0.2.951.dist-info/METADATA +0 -61
- pybiolib-0.2.951.dist-info/RECORD +0 -153
- pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
- /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
biolib/cli_utils.py
DELETED
|
@@ -1,273 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import shutil
|
|
3
|
-
import sys
|
|
4
|
-
import platform
|
|
5
|
-
import tempfile
|
|
6
|
-
import time
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
from typing import Tuple
|
|
9
|
-
from zipfile import ZipFile
|
|
10
|
-
from termcolor import colored
|
|
11
|
-
|
|
12
|
-
import yaml
|
|
13
|
-
|
|
14
|
-
from biolib.biolib_api_client import AppVersionOnJob, Module, FilesMapping
|
|
15
|
-
from biolib.validators.validate_app_version import validate_app_version # type: ignore
|
|
16
|
-
from biolib.validators.validate_argument import validate_argument # type: ignore
|
|
17
|
-
from biolib.validators.validate_module import validate_module # type: ignore
|
|
18
|
-
from biolib.biolib_errors import BioLibError
|
|
19
|
-
from biolib.biolib_logging import logger
|
|
20
|
-
from biolib.typing_utils import Dict, List
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class BiolibValidationError(Exception):
|
|
24
|
-
pass
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def bl_colored(string, colour):
|
|
28
|
-
# The colors does not seem to play nice with Windows PowerShell terminal
|
|
29
|
-
if platform.system() == 'Windows':
|
|
30
|
-
return string
|
|
31
|
-
else:
|
|
32
|
-
return colored(string, colour)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def validate_and_get_args(args):
|
|
36
|
-
if args is None:
|
|
37
|
-
args = []
|
|
38
|
-
|
|
39
|
-
if isinstance(args, str):
|
|
40
|
-
args = list(filter(lambda p: p != '', args.split(' ')))
|
|
41
|
-
|
|
42
|
-
if not isinstance(args, list):
|
|
43
|
-
raise Exception('The given input arguments must be list or str')
|
|
44
|
-
|
|
45
|
-
return args
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def get_files_dict_and_file_args(files, args):
|
|
49
|
-
if files is None:
|
|
50
|
-
files = []
|
|
51
|
-
# TODO: Figure out how to make this slightly less error prone
|
|
52
|
-
for idx, arg in enumerate(args):
|
|
53
|
-
if not arg.startswith('/'):
|
|
54
|
-
arg = f'{os.getcwd()}/{arg}'
|
|
55
|
-
|
|
56
|
-
arg = Path(arg)
|
|
57
|
-
if arg.is_file():
|
|
58
|
-
files.append(arg)
|
|
59
|
-
# Make sure that arg is only the filename
|
|
60
|
-
args[idx] = arg.parts[-1]
|
|
61
|
-
|
|
62
|
-
files_dict = {}
|
|
63
|
-
for file_path in files:
|
|
64
|
-
file = open(file_path, 'rb')
|
|
65
|
-
path = '/' + file_path.parts[-1]
|
|
66
|
-
|
|
67
|
-
files_dict[path] = file.read()
|
|
68
|
-
file.close()
|
|
69
|
-
|
|
70
|
-
return files_dict, args
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def validate_and_get_app_path(provided_app_path):
|
|
74
|
-
if provided_app_path.startswith('/'):
|
|
75
|
-
app_path = Path(provided_app_path)
|
|
76
|
-
else:
|
|
77
|
-
app_path = Path(os.getcwd()) / provided_app_path
|
|
78
|
-
|
|
79
|
-
if not app_path.exists():
|
|
80
|
-
print(f'ERROR: Could not find path {app_path}')
|
|
81
|
-
sys.exit(1)
|
|
82
|
-
|
|
83
|
-
if not app_path.is_dir():
|
|
84
|
-
print(f'ERROR: Path {app_path} is not a directory')
|
|
85
|
-
sys.exit(1)
|
|
86
|
-
|
|
87
|
-
return app_path
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
def get_yaml_data(app_path):
|
|
91
|
-
try:
|
|
92
|
-
yaml_file = open(f'{app_path}/.biolib/config.yml', 'r', encoding='utf-8')
|
|
93
|
-
|
|
94
|
-
except Exception as error: # pylint: disable=broad-except
|
|
95
|
-
raise BioLibError(f'Could not open the biolib config file at {app_path}/.biolib/config.yml') from error
|
|
96
|
-
|
|
97
|
-
try:
|
|
98
|
-
yaml_data = yaml.safe_load(yaml_file)
|
|
99
|
-
|
|
100
|
-
except Exception as error: # pylint: disable=broad-except
|
|
101
|
-
raise BioLibError(f'Could not parse {app_path}/.biolib/config.yml. Please make sure it is valid YAML') \
|
|
102
|
-
from error
|
|
103
|
-
|
|
104
|
-
return yaml_data
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def validate_arguments(yaml_data):
|
|
108
|
-
error_dict = {'arguments': {}}
|
|
109
|
-
# Arguments are optional
|
|
110
|
-
if 'arguments' in yaml_data:
|
|
111
|
-
for argument in yaml_data['arguments']:
|
|
112
|
-
argument_errors = validate_argument(argument)
|
|
113
|
-
if argument_errors:
|
|
114
|
-
error_dict['arguments'].update(argument_errors)
|
|
115
|
-
|
|
116
|
-
# Just return empty dict if we have no errors
|
|
117
|
-
if error_dict['arguments']:
|
|
118
|
-
return error_dict
|
|
119
|
-
else:
|
|
120
|
-
return {}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def validate_and_get_biolib_yaml_version(yaml_data, error_dict):
|
|
124
|
-
if not 'biolib_version' in yaml_data.keys():
|
|
125
|
-
error_dict['config_yml']['biolib_version'] = ['Your config file is missing the biolib_version field.']
|
|
126
|
-
print_errors(error_dict)
|
|
127
|
-
sys.exit(1)
|
|
128
|
-
else:
|
|
129
|
-
biolib_version = yaml_data['biolib_version']
|
|
130
|
-
|
|
131
|
-
if biolib_version not in (1, 2):
|
|
132
|
-
error_dict['config_yml']['biolib_version'] = \
|
|
133
|
-
[f'Biolib version can only be either 1 or 2 for now, your version is {biolib_version}.']
|
|
134
|
-
print_errors(error_dict)
|
|
135
|
-
sys.exit(1)
|
|
136
|
-
|
|
137
|
-
return biolib_version
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
def validate_modules(yaml_data, biolib_yaml_version):
|
|
141
|
-
error_dict = {'modules': {}}
|
|
142
|
-
if 'modules' in yaml_data:
|
|
143
|
-
for name, task_data in yaml_data['modules'].items():
|
|
144
|
-
task_errors = validate_module(name, task_data, biolib_yaml_version)
|
|
145
|
-
if task_errors:
|
|
146
|
-
error_dict['modules'].update(task_errors)
|
|
147
|
-
|
|
148
|
-
# Just return empty dict if we have no errors
|
|
149
|
-
if error_dict['modules']:
|
|
150
|
-
return error_dict
|
|
151
|
-
else:
|
|
152
|
-
return {}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
def validate_yaml_config(yaml_data, source_files_zip_path):
|
|
156
|
-
source_files_zip = ZipFile(source_files_zip_path)
|
|
157
|
-
error_dict = {'config_yml': {}}
|
|
158
|
-
|
|
159
|
-
biolib_yaml_version = validate_and_get_biolib_yaml_version(yaml_data, error_dict)
|
|
160
|
-
error_dict['config_yml'].update(validate_app_version(yaml_data, biolib_yaml_version, source_files_zip))
|
|
161
|
-
error_dict['config_yml'].update(validate_modules(yaml_data, biolib_yaml_version))
|
|
162
|
-
error_dict['config_yml'].update(validate_arguments(yaml_data))
|
|
163
|
-
|
|
164
|
-
if error_dict['config_yml']:
|
|
165
|
-
print_errors(error_dict)
|
|
166
|
-
raise BiolibValidationError('Could not validate .biolib/config.yml')
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def print_errors(error_dict):
|
|
170
|
-
print(bl_colored("\nThe following validation errors were found in the .biolib/config.yml file:", 'red'))
|
|
171
|
-
print(yaml.safe_dump(error_dict, allow_unicode=True, default_flow_style=False))
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
def write_output_files(mapped_output_files=None):
|
|
175
|
-
output_dir_path = 'biolib_results'
|
|
176
|
-
|
|
177
|
-
if os.path.exists(output_dir_path):
|
|
178
|
-
os.rename(output_dir_path, f'{output_dir_path}_old_{time.strftime("%Y%m%d%H%M%S")}')
|
|
179
|
-
|
|
180
|
-
if mapped_output_files:
|
|
181
|
-
logger.debug("Output Files:")
|
|
182
|
-
|
|
183
|
-
for path, data in mapped_output_files.items():
|
|
184
|
-
dirs, file = os.path.split(path)
|
|
185
|
-
path_on_disk = f'{output_dir_path}{path}'
|
|
186
|
-
|
|
187
|
-
if dirs:
|
|
188
|
-
os.makedirs(f'{output_dir_path}{dirs}', exist_ok=True)
|
|
189
|
-
if file:
|
|
190
|
-
open(path_on_disk, 'wb').write(data)
|
|
191
|
-
|
|
192
|
-
logger.debug(f" - {path_on_disk}")
|
|
193
|
-
|
|
194
|
-
logger.debug('\n')
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
def get_pretty_print_module_output_string(stdout=None, stderr=None, exitcode=None) -> str:
|
|
198
|
-
stderr_string = f"\n{bl_colored('stderr:', 'red')}\n{stderr.decode()}" if stderr else ''
|
|
199
|
-
return f'''{bl_colored('stdout:', 'green')}
|
|
200
|
-
{stdout.decode()}{stderr_string}
|
|
201
|
-
{bl_colored('exitcode:', 'green' if exitcode == 0 else 'red')} {exitcode}'''
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def _get_files_mappings(file_commands: List[str]) -> List[FilesMapping]:
|
|
205
|
-
files_mappings: List[FilesMapping] = []
|
|
206
|
-
for file_command in file_commands:
|
|
207
|
-
# TODO: figure out if splitting on space is good enough
|
|
208
|
-
_, from_path, to_path = file_command.split(' ')
|
|
209
|
-
files_mappings.append(FilesMapping(from_path=from_path, to_path=to_path))
|
|
210
|
-
return files_mappings
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
def get_mocked_app_version_from_yaml(yaml_data: Dict, source_files_zip_path: str):
|
|
214
|
-
modules: List[Module] = []
|
|
215
|
-
for module_name in yaml_data['modules']:
|
|
216
|
-
yaml_module = yaml_data['modules'][module_name]
|
|
217
|
-
|
|
218
|
-
image_hostname, image_uri = yaml_module['image'].split('://')
|
|
219
|
-
if image_hostname != 'local-docker':
|
|
220
|
-
raise Exception(
|
|
221
|
-
f'Found image {image_uri} with hostname {image_hostname}, please use "local-docker://"'
|
|
222
|
-
f'Your application must use modules with local docker images to use run-dev'
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
modules.append(Module(
|
|
226
|
-
command=yaml_module.get('command', ''),
|
|
227
|
-
environment='biolib-ecr',
|
|
228
|
-
image_uri=image_uri,
|
|
229
|
-
input_files_mappings=_get_files_mappings(yaml_module['input_files']),
|
|
230
|
-
large_file_systems=[],
|
|
231
|
-
name=module_name,
|
|
232
|
-
output_files_mappings=_get_files_mappings(yaml_module['output_files']),
|
|
233
|
-
source_files_mappings=_get_files_mappings(yaml_module['source_files']),
|
|
234
|
-
working_directory=yaml_module['working_directory'],
|
|
235
|
-
estimated_image_size_bytes=None # This should not be provided here
|
|
236
|
-
))
|
|
237
|
-
|
|
238
|
-
return AppVersionOnJob(
|
|
239
|
-
client_side_executable_zip=source_files_zip_path,
|
|
240
|
-
consumes_stdin=True,
|
|
241
|
-
is_runnable_by_user=True,
|
|
242
|
-
modules=modules,
|
|
243
|
-
public_id='app-version-mock-id',
|
|
244
|
-
remote_hosts=[],
|
|
245
|
-
settings=[],
|
|
246
|
-
stdout_render_type='text',
|
|
247
|
-
)
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
def create_temporary_directory_with_source_files_zip(app_path: str) -> Tuple[tempfile.TemporaryDirectory, str]:
|
|
251
|
-
original_working_directory = os.getcwd()
|
|
252
|
-
temporary_directory = tempfile.TemporaryDirectory()
|
|
253
|
-
os.chdir(app_path)
|
|
254
|
-
try:
|
|
255
|
-
zip_path_without_file_extension = os.path.join(temporary_directory.name, 'biolib-cli-tmp-source-files-zip')
|
|
256
|
-
zip_path = f'{zip_path_without_file_extension}.zip'
|
|
257
|
-
app_folder_name = Path(app_path).absolute().parts[-1]
|
|
258
|
-
shutil.make_archive(
|
|
259
|
-
base_name=zip_path_without_file_extension,
|
|
260
|
-
format='zip',
|
|
261
|
-
root_dir='..',
|
|
262
|
-
base_dir=app_folder_name,
|
|
263
|
-
)
|
|
264
|
-
|
|
265
|
-
return temporary_directory, zip_path
|
|
266
|
-
|
|
267
|
-
except Exception as error:
|
|
268
|
-
temporary_directory.cleanup()
|
|
269
|
-
raise Exception('Failed to create zip of application') from error
|
|
270
|
-
|
|
271
|
-
finally:
|
|
272
|
-
# change back to old working directory
|
|
273
|
-
os.chdir(original_working_directory)
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
from urllib.parse import urlparse
|
|
2
|
-
from biolib.biolib_api_client import RemoteHost
|
|
3
|
-
from biolib.biolib_docker_client import BiolibDockerClient
|
|
4
|
-
from biolib.biolib_logging import logger
|
|
5
|
-
from biolib.compute_node.remote_host_proxy import RemoteHostProxy
|
|
6
|
-
from biolib.compute_node.webserver.webserver_types import WebserverConfig
|
|
7
|
-
from biolib.typing_utils import List
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def start_enclave_remote_hosts(config: WebserverConfig) -> None:
|
|
12
|
-
logger.debug('Starting Docker network for enclave remote host proxies')
|
|
13
|
-
docker = BiolibDockerClient.get_docker_client()
|
|
14
|
-
public_network = docker.networks.create(
|
|
15
|
-
driver='bridge',
|
|
16
|
-
internal=False,
|
|
17
|
-
name='biolib-enclave-remote-hosts-network',
|
|
18
|
-
)
|
|
19
|
-
|
|
20
|
-
biolib_remote_hosts: List[RemoteHost] = []
|
|
21
|
-
biolib_remote_host_proxies: List[RemoteHostProxy] = []
|
|
22
|
-
base_hostname = urlparse(config['base_url']).hostname
|
|
23
|
-
# Make sure base_hostname is not None for typing reasons
|
|
24
|
-
if not base_hostname:
|
|
25
|
-
raise Exception('Base hostname not set, likely due to base url not being set. This is required in enclaves')
|
|
26
|
-
|
|
27
|
-
if base_hostname == 'biolib.com':
|
|
28
|
-
biolib_remote_hosts.append(RemoteHost(hostname='containers.biolib.com'))
|
|
29
|
-
elif base_hostname == 'staging-elb.biolib.com':
|
|
30
|
-
biolib_remote_hosts.append(RemoteHost(hostname='containers.staging.biolib.com'))
|
|
31
|
-
|
|
32
|
-
# Allow reaching backend for self registering and job creation
|
|
33
|
-
biolib_remote_hosts.append(RemoteHost(hostname=base_hostname))
|
|
34
|
-
# For downloading container image layers from S3 URLs returned by ECR proxy
|
|
35
|
-
biolib_remote_hosts.append(RemoteHost(
|
|
36
|
-
hostname=f"prod-{config['ecr_region_name']}-starport-layer-bucket.s3.{config['ecr_region_name']}.amazonaws.com"
|
|
37
|
-
))
|
|
38
|
-
# For downloading source files zip
|
|
39
|
-
biolib_remote_hosts.append(RemoteHost(
|
|
40
|
-
hostname=f"{config['s3_general_storage_bucket_name']}.s3.amazonaws.com"
|
|
41
|
-
))
|
|
42
|
-
|
|
43
|
-
logger.debug('Starting enclave remote host proxies')
|
|
44
|
-
for remote_host in biolib_remote_hosts:
|
|
45
|
-
remote_host_proxy = RemoteHostProxy(remote_host, public_network, internal_network=None, job_id=None)
|
|
46
|
-
remote_host_proxy.start()
|
|
47
|
-
biolib_remote_host_proxies.append(remote_host_proxy)
|
|
48
|
-
|
|
49
|
-
logger.debug('Writing to enclave /etc/hosts')
|
|
50
|
-
with open('/etc/hosts', mode='a') as hosts_file:
|
|
51
|
-
for proxy in biolib_remote_host_proxies:
|
|
52
|
-
ip_address = proxy.get_ip_address_on_network(public_network)
|
|
53
|
-
hosts_file.write(f'\n{ip_address} {proxy.hostname}')
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
This file is modified based on donkersgoed's repository (https://github.com/donkersgoed/nitropepper-enclave-app)
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
import Crypto
|
|
6
|
-
from Crypto.PublicKey import RSA
|
|
7
|
-
from Crypto.Cipher import PKCS1_OAEP
|
|
8
|
-
from Crypto.Hash import SHA256
|
|
9
|
-
|
|
10
|
-
# enclave_build will only be present when running on an enclave
|
|
11
|
-
import biolib.compute_node.enclave.libnsm as libnsm # type: ignore # pylint: disable=import-error, no-name-in-module
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class NitroSecureModuleUtils:
|
|
15
|
-
"""NSM util class."""
|
|
16
|
-
|
|
17
|
-
def __init__(self):
|
|
18
|
-
"""Construct a new NSMUtil instance."""
|
|
19
|
-
# Initialize the Rust NSM Library
|
|
20
|
-
self._nsm_fd = libnsm.nsm_lib_init() # pylint:disable=c-extension-no-member
|
|
21
|
-
# Create a new random function `nsm_rand_func`, which
|
|
22
|
-
# utilizes the NSM module.
|
|
23
|
-
self.nsm_rand_func = lambda num_bytes: libnsm.nsm_get_random( # pylint:disable=c-extension-no-member
|
|
24
|
-
self._nsm_fd, num_bytes
|
|
25
|
-
)
|
|
26
|
-
|
|
27
|
-
# Force pycryptodome to use the new rand function.
|
|
28
|
-
# Without this, pycryptodome defaults to /dev/random
|
|
29
|
-
# and /dev/urandom, which are not available in Enclaves.
|
|
30
|
-
self._monkey_patch_crypto(self.nsm_rand_func)
|
|
31
|
-
|
|
32
|
-
# Generate a new RSA certificate, which will be used to
|
|
33
|
-
# generate the Attestation document and to decrypt results
|
|
34
|
-
# for KMS Decrypt calls with this document.
|
|
35
|
-
self._rsa_key = RSA.generate(2048)
|
|
36
|
-
self._public_key = self._rsa_key.publickey().export_key('DER')
|
|
37
|
-
|
|
38
|
-
def get_attestation_doc(self):
|
|
39
|
-
"""Get the attestation document from /dev/nsm."""
|
|
40
|
-
libnsm_att_doc_cose_signed = libnsm.nsm_get_attestation_doc( # pylint:disable=c-extension-no-member
|
|
41
|
-
self._nsm_fd,
|
|
42
|
-
self._public_key,
|
|
43
|
-
len(self._public_key)
|
|
44
|
-
)
|
|
45
|
-
return libnsm_att_doc_cose_signed
|
|
46
|
-
|
|
47
|
-
def decrypt(self, ciphertext):
|
|
48
|
-
"""
|
|
49
|
-
Decrypt ciphertext using private key
|
|
50
|
-
"""
|
|
51
|
-
cipher = PKCS1_OAEP.new(self._rsa_key, hashAlgo=SHA256)
|
|
52
|
-
plaintext = cipher.decrypt(ciphertext)
|
|
53
|
-
|
|
54
|
-
return plaintext
|
|
55
|
-
|
|
56
|
-
@classmethod
|
|
57
|
-
def _monkey_patch_crypto(cls, nsm_rand_func):
|
|
58
|
-
"""Monkeypatch Crypto to use the NSM rand function."""
|
|
59
|
-
Crypto.Random.get_random_bytes = nsm_rand_func
|
|
60
|
-
|
|
61
|
-
def new_random_read(self, n_bytes): # pylint:disable=unused-argument
|
|
62
|
-
return nsm_rand_func(n_bytes)
|
|
63
|
-
|
|
64
|
-
Crypto.Random._UrandomRNG.read = new_random_read # pylint:disable=protected-access
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import abc
|
|
2
|
-
|
|
3
|
-
from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class BaseExecutor(abc.ABC):
|
|
7
|
-
|
|
8
|
-
def __init__(self, options: LocalExecutorOptions):
|
|
9
|
-
self._options: LocalExecutorOptions = options
|
|
10
|
-
self._is_cleaning_up = False
|
|
11
|
-
|
|
12
|
-
@abc.abstractmethod
|
|
13
|
-
def execute_module(self, module_input_serialized: bytes) -> bytes:
|
|
14
|
-
pass
|
|
15
|
-
|
|
16
|
-
@abc.abstractmethod
|
|
17
|
-
def cleanup(self) -> None:
|
|
18
|
-
pass
|
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
import http.server
|
|
2
|
-
import json
|
|
3
|
-
import socketserver
|
|
4
|
-
import threading
|
|
5
|
-
import os
|
|
6
|
-
import asyncio
|
|
7
|
-
import subprocess
|
|
8
|
-
import base64
|
|
9
|
-
import sys
|
|
10
|
-
import logging
|
|
11
|
-
|
|
12
|
-
from biolib.typing_utils import Optional
|
|
13
|
-
from biolib import utils
|
|
14
|
-
from biolib.biolib_api_client import BiolibApiClient
|
|
15
|
-
from biolib.biolib_errors import BioLibError
|
|
16
|
-
from biolib.biolib_logging import logger, TRACE
|
|
17
|
-
from biolib.compute_node.job_worker.executors.base_executor import BaseExecutor
|
|
18
|
-
from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions
|
|
19
|
-
from biolib.pyppeteer.pyppeteer import launch, command # type: ignore
|
|
20
|
-
from biolib.pyppeteer.pyppeteer.launcher import resolveExecutablePath, __chromium_revision__ # type: ignore
|
|
21
|
-
|
|
22
|
-
# specifically limit logs from pyppeteer
|
|
23
|
-
logging.getLogger('biolib.pyppeteer.pyppeteer').setLevel(logging.ERROR)
|
|
24
|
-
logging.getLogger('biolib.pyppeteer.pyppeteer.connection').setLevel(logging.ERROR)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class RequestHandler(http.server.SimpleHTTPRequestHandler):
|
|
28
|
-
def log_message(self, format, *args): # pylint: disable=redefined-builtin
|
|
29
|
-
if logger.level == TRACE:
|
|
30
|
-
http.server.SimpleHTTPRequestHandler.log_message(self, format, *args)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
class PyppeteerExecutor(BaseExecutor):
|
|
34
|
-
_chrome_executable_path: Optional[str] = None
|
|
35
|
-
_no_sandbox: bool = True
|
|
36
|
-
|
|
37
|
-
def execute_module(self, module_input_serialized: bytes) -> bytes:
|
|
38
|
-
with socketserver.TCPServer(('127.0.0.1', 0), RequestHandler) as httpd:
|
|
39
|
-
port = httpd.server_address[1]
|
|
40
|
-
thread = threading.Thread(target=httpd.serve_forever)
|
|
41
|
-
original_working_directory = os.getcwd()
|
|
42
|
-
# TODO: figure out how we can avoid changing the current directory
|
|
43
|
-
biolib_js_dir = os.path.join(utils.BIOLIB_PACKAGE_ROOT_DIR, 'biolib-js')
|
|
44
|
-
os.chdir(biolib_js_dir)
|
|
45
|
-
try:
|
|
46
|
-
thread.start()
|
|
47
|
-
module_output_serialized: bytes = asyncio.get_event_loop().run_until_complete(self._call_pyppeteer(
|
|
48
|
-
port,
|
|
49
|
-
self._options,
|
|
50
|
-
module_input_serialized,
|
|
51
|
-
))
|
|
52
|
-
return module_output_serialized
|
|
53
|
-
finally:
|
|
54
|
-
os.chdir(original_working_directory)
|
|
55
|
-
httpd.shutdown()
|
|
56
|
-
thread.join()
|
|
57
|
-
|
|
58
|
-
async def _call_pyppeteer(self, port: int, options: LocalExecutorOptions, module_input_serialized: bytes):
|
|
59
|
-
if not self._chrome_executable_path:
|
|
60
|
-
mac_chrome_path = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
|
|
61
|
-
if os.path.isfile(mac_chrome_path):
|
|
62
|
-
self._chrome_executable_path = mac_chrome_path
|
|
63
|
-
|
|
64
|
-
if not self._chrome_executable_path:
|
|
65
|
-
linux_chrome_path = '/usr/lib/chromium-browser/chromium-browser'
|
|
66
|
-
|
|
67
|
-
# special install for google colab
|
|
68
|
-
if not os.path.isfile(linux_chrome_path) and 'google.colab' in sys.modules:
|
|
69
|
-
subprocess.run('apt-get update', shell=True, check=True)
|
|
70
|
-
subprocess.run('apt install chromium-chromedriver', shell=True, check=True)
|
|
71
|
-
|
|
72
|
-
if os.path.isfile(linux_chrome_path):
|
|
73
|
-
self._chrome_executable_path = linux_chrome_path
|
|
74
|
-
|
|
75
|
-
resolved_path = resolveExecutablePath(None, __chromium_revision__)
|
|
76
|
-
if not self._chrome_executable_path and resolved_path[1]:
|
|
77
|
-
# if executable_path is not set explicit,
|
|
78
|
-
# and resolveExecutablePath failed (== we got an error message back in resolved_path[1])
|
|
79
|
-
logger.info('Installing dependencies...')
|
|
80
|
-
os.environ['PYPPETEER_NO_PROGRESS_BAR'] = 'true'
|
|
81
|
-
command.install()
|
|
82
|
-
|
|
83
|
-
logger.info('Computing...')
|
|
84
|
-
|
|
85
|
-
chrome_arguments = [
|
|
86
|
-
'--disable-web-security',
|
|
87
|
-
]
|
|
88
|
-
if self._no_sandbox:
|
|
89
|
-
chrome_arguments.append('--no-sandbox')
|
|
90
|
-
|
|
91
|
-
browser = await launch(args=chrome_arguments, executablePath=self._chrome_executable_path)
|
|
92
|
-
|
|
93
|
-
# start new page
|
|
94
|
-
page = await browser.newPage()
|
|
95
|
-
|
|
96
|
-
await page.goto('http://localhost:' + str(port))
|
|
97
|
-
|
|
98
|
-
def get_data():
|
|
99
|
-
return base64.b64encode(module_input_serialized).decode('ascii')
|
|
100
|
-
|
|
101
|
-
def set_progress_compute(value):
|
|
102
|
-
logger.debug(f'Compute progress: {value}')
|
|
103
|
-
|
|
104
|
-
def set_progress_initialization(value):
|
|
105
|
-
logger.debug(f'Initialization progress: {value}')
|
|
106
|
-
|
|
107
|
-
def add_log_message(value):
|
|
108
|
-
logger.debug(f'Log message: {value}')
|
|
109
|
-
|
|
110
|
-
await page.exposeFunction('getData', get_data)
|
|
111
|
-
await page.exposeFunction('setProgressCompute', set_progress_compute)
|
|
112
|
-
await page.exposeFunction('setProgressInitialization', set_progress_initialization)
|
|
113
|
-
await page.exposeFunction('addLogMessage', add_log_message)
|
|
114
|
-
|
|
115
|
-
api_client = BiolibApiClient.get()
|
|
116
|
-
refresh_token_js_value = 'undefined' if not api_client.refresh_token else f'"{api_client.refresh_token}"'
|
|
117
|
-
|
|
118
|
-
job = options['job'].copy()
|
|
119
|
-
if 'custom_compute_node_url' in job:
|
|
120
|
-
job.pop('custom_compute_node_url')
|
|
121
|
-
|
|
122
|
-
job_json_string = json.dumps(job)
|
|
123
|
-
|
|
124
|
-
output_serialized_js_bytes = await page.evaluate('''
|
|
125
|
-
async function() {
|
|
126
|
-
const refreshToken = ''' + refresh_token_js_value + ''';
|
|
127
|
-
const baseUrl = \'''' + options['biolib_base_url'] + '''\';
|
|
128
|
-
const job = ''' + job_json_string + ''';
|
|
129
|
-
|
|
130
|
-
const { BioLibSingleton, AppClient } = window.BioLib;
|
|
131
|
-
BioLibSingleton.setConfig({ baseUrl, refreshToken });
|
|
132
|
-
AppClient.setApiClient(BioLibSingleton.get());
|
|
133
|
-
|
|
134
|
-
const inputBase64 = await window.getData();
|
|
135
|
-
const inputByteArray = Uint8Array.from(atob(inputBase64), c => c.charCodeAt(0));
|
|
136
|
-
|
|
137
|
-
const jobUtils = {
|
|
138
|
-
setProgressCompute: window.setProgressCompute,
|
|
139
|
-
setProgressInitialization: window.setProgressInitialization,
|
|
140
|
-
addLogMessage: window.addLogMessage,
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
try {
|
|
144
|
-
const moduleOutput = await AppClient.runJob(job, inputByteArray, jobUtils);
|
|
145
|
-
return moduleOutput.serialize();
|
|
146
|
-
} catch(err) {
|
|
147
|
-
return err.toString();
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
''')
|
|
151
|
-
logger.debug('Closing browser')
|
|
152
|
-
await browser.close()
|
|
153
|
-
|
|
154
|
-
module_output_serialized = self._js_to_python_byte_array_converter(output_serialized_js_bytes)
|
|
155
|
-
|
|
156
|
-
# TODO: check if module_output is correctly serialized
|
|
157
|
-
if isinstance(module_output_serialized, bytes):
|
|
158
|
-
return module_output_serialized
|
|
159
|
-
else:
|
|
160
|
-
raise BioLibError(module_output_serialized)
|
|
161
|
-
|
|
162
|
-
def cleanup(self):
|
|
163
|
-
pass
|
|
164
|
-
|
|
165
|
-
@staticmethod
|
|
166
|
-
def _js_to_python_byte_array_converter(js_encoded) -> bytes:
|
|
167
|
-
try:
|
|
168
|
-
return bytes(list([js_encoded[str(i)] for i in range(len(js_encoded))]))
|
|
169
|
-
except Exception as error:
|
|
170
|
-
logger.error('Failed to decode response from browser')
|
|
171
|
-
logger.error(error)
|
|
172
|
-
logger.error(js_encoded)
|
|
173
|
-
raise BioLibError(js_encoded) from error
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from biolib.compute_node.job_worker.executors.remote.remote_executor import RemoteExecutor
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import cbor2 # type: ignore
|
|
2
|
-
import cose # type: ignore
|
|
3
|
-
|
|
4
|
-
from cose import EC2, CoseAlgorithms, CoseEllipticCurves
|
|
5
|
-
from Crypto.Util.number import long_to_bytes
|
|
6
|
-
from OpenSSL import crypto
|
|
7
|
-
|
|
8
|
-
from biolib import utils
|
|
9
|
-
from biolib.biolib_errors import BioLibError
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class NitroEnclaveUtils:
|
|
13
|
-
def attest_enclave_and_get_rsa_public_key(self, expected_pcrs_and_aws_cert, attestation_document_bytes):
|
|
14
|
-
cbor_data = cbor2.loads(attestation_document_bytes)
|
|
15
|
-
cbor_document = cbor2.loads(cbor_data[2])
|
|
16
|
-
expected_pcrs_list = expected_pcrs_and_aws_cert['pcrObjectArray']
|
|
17
|
-
aws_nitro_root_cert_pem = "-----BEGIN CERTIFICATE-----\n" + \
|
|
18
|
-
expected_pcrs_and_aws_cert['awsNitroRootCertificateBase64'] + \
|
|
19
|
-
"\n-----END CERTIFICATE-----"
|
|
20
|
-
actual_pcrs = {str(k): v.hex() for k, v in cbor_document['pcrs'].items()}
|
|
21
|
-
|
|
22
|
-
if not utils.BIOLIB_CLOUD_SKIP_PCR_VERIFICATION:
|
|
23
|
-
self._assert_pcr_validity(actual_pcrs, expected_pcrs_list)
|
|
24
|
-
|
|
25
|
-
cert = crypto.load_certificate(crypto.FILETYPE_ASN1, cbor_document['certificate'])
|
|
26
|
-
self._assert_certificate_chain_validity(cbor_document['cabundle'], cert, aws_nitro_root_cert_pem)
|
|
27
|
-
self._assert_cose_signature_validity(cert, cbor_data)
|
|
28
|
-
return cbor_document['public_key']
|
|
29
|
-
|
|
30
|
-
def _assert_pcr_validity(self, actual_pcrs, expected_pcrs_list):
|
|
31
|
-
for expected_pcrs in expected_pcrs_list:
|
|
32
|
-
if self._does_pcr_object_match_actual(actual_pcrs, expected_pcrs):
|
|
33
|
-
return True
|
|
34
|
-
|
|
35
|
-
# Raise error if no match was fund
|
|
36
|
-
raise BioLibError('Failed to verify PCRs')
|
|
37
|
-
|
|
38
|
-
def _does_pcr_object_match_actual(self, actual_pcrs, expected_pcrs):
|
|
39
|
-
for pcr_key in expected_pcrs.keys():
|
|
40
|
-
if actual_pcrs[pcr_key] != expected_pcrs[pcr_key]:
|
|
41
|
-
return False
|
|
42
|
-
# If all matched return True
|
|
43
|
-
return True
|
|
44
|
-
|
|
45
|
-
def _assert_certificate_chain_validity(self, cabundle, cert, aws_nitro_root_cert_pem):
|
|
46
|
-
store = crypto.X509Store()
|
|
47
|
-
_cert = crypto.load_certificate(crypto.FILETYPE_PEM, aws_nitro_root_cert_pem)
|
|
48
|
-
store.add_cert(_cert)
|
|
49
|
-
|
|
50
|
-
for _cert_binary in cabundle[1:]:
|
|
51
|
-
_cert = crypto.load_certificate(crypto.FILETYPE_ASN1, _cert_binary)
|
|
52
|
-
store.add_cert(_cert)
|
|
53
|
-
|
|
54
|
-
store_ctx = crypto.X509StoreContext(store, cert)
|
|
55
|
-
try:
|
|
56
|
-
store_ctx.verify_certificate()
|
|
57
|
-
except Exception as exception:
|
|
58
|
-
raise BioLibError('Failed to verify certificates') from exception
|
|
59
|
-
|
|
60
|
-
def _assert_cose_signature_validity(self, cert, cbor_data):
|
|
61
|
-
phdr = cbor2.loads(cbor_data[0])
|
|
62
|
-
uhdr = cbor_data[1]
|
|
63
|
-
signature = cbor_data[3]
|
|
64
|
-
|
|
65
|
-
cert_public_numbers = cert.get_pubkey().to_cryptography_key().public_numbers()
|
|
66
|
-
x_long = cert_public_numbers.x
|
|
67
|
-
y_long = cert_public_numbers.y
|
|
68
|
-
|
|
69
|
-
x_bytes = long_to_bytes(x_long)
|
|
70
|
-
y_bytes = long_to_bytes(y_long)
|
|
71
|
-
|
|
72
|
-
# Create the EC2 key from public key parameters
|
|
73
|
-
key = EC2(alg=CoseAlgorithms.ES384, x=x_bytes, y=y_bytes, crv=CoseEllipticCurves.P_384)
|
|
74
|
-
|
|
75
|
-
# Construct the Sign1 message
|
|
76
|
-
msg = cose.Sign1Message(phdr=phdr, uhdr=uhdr, payload=cbor_data[2])
|
|
77
|
-
msg.signature = signature
|
|
78
|
-
|
|
79
|
-
# Verify the signature using the EC2 key
|
|
80
|
-
if not msg.verify_signature(key):
|
|
81
|
-
raise Exception('Failed to verify signature in attestation document')
|