pybiolib 1.1.1881__py3-none-any.whl → 1.1.2193__py3-none-any.whl

biolib/compute_node/remote_host_proxy.py

@@ -1,21 +1,21 @@
 import io
-import tarfile
 import subprocess
+import tarfile
 import time
+from urllib.parse import urlparse
 
-from docker.models.containers import Container  # type: ignore
 from docker.errors import ImageNotFound  # type: ignore
+from docker.models.containers import Container  # type: ignore
 from docker.models.images import Image  # type: ignore
 from docker.models.networks import Network  # type: ignore
 
 from biolib import utils
-from biolib.biolib_errors import BioLibError
-from biolib.compute_node.cloud_utils import CloudUtils
-from biolib.typing_utils import Optional, List
-from biolib.biolib_api_client import RemoteHost
+from biolib.biolib_api_client import BiolibApiClient, RemoteHost
 from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger_no_user_data
-from biolib.biolib_api_client import BiolibApiClient
+from biolib.compute_node.cloud_utils import CloudUtils
+from biolib.typing_utils import List, Optional
 
 
 # Prepare for remote hosts with specified port
@@ -24,31 +24,23 @@ class RemoteHostExtended(RemoteHost):
 
 
 class RemoteHostProxy:
-
     def __init__(
-            self,
-            remote_host: RemoteHost,
-            public_network: Network,
-            internal_network: Optional[Network],
-            job_id: str,
-            ports: List[int],
-            can_push_data_record_for_user: bool,
+        self,
+        remote_host: RemoteHost,
+        public_network: Network,
+        internal_network: Optional[Network],
+        job_id: str,
+        ports: List[int],
     ):
-        self._can_push_data_record_for_user: bool = can_push_data_record_for_user
         self.is_app_caller_proxy = remote_host['hostname'] == 'AppCallerProxy'
-
-        # Default to port 443 for now until backend serves remote_hosts with port specified
-        self._remote_host: RemoteHostExtended = RemoteHostExtended(
-            hostname=remote_host['hostname'],
-            ports=ports
-        )
+        self._remote_host: RemoteHostExtended = RemoteHostExtended(hostname=remote_host['hostname'], ports=ports)
         self._public_network: Network = public_network
         self._internal_network: Optional[Network] = internal_network
 
         if not job_id:
             raise Exception('RemoteHostProxy missing argument "job_id"')
 
-        self._name = f"biolib-remote-host-proxy-{job_id}-{self.hostname}"
+        self._name = f'biolib-remote-host-proxy-{job_id}-{self.hostname}'
         self._job_uuid = job_id
         self._container: Optional[Container] = None
         self._enclave_traffic_forwarder_processes: List[subprocess.Popen] = []
@@ -154,32 +146,21 @@ class RemoteHostProxy:
             raise Exception('RemoteHostProxy container not defined when attempting to write NGINX config')
 
         docker = BiolibDockerClient.get_docker_client()
-        base_url = BiolibApiClient.get().base_url
+        upstream_hostname = urlparse(BiolibApiClient.get().base_url).hostname
         if self.is_app_caller_proxy:
+            if not utils.IS_RUNNING_IN_CLOUD:
+                raise BioLibError('Calling apps inside apps is not supported in local compute environment')
+
             logger_no_user_data.debug(f'Job "{self._job_uuid}" writing config for and starting App Caller Proxy...')
-            if utils.BIOLIB_CLOUD_BASE_URL:
-                cloud_base_url = utils.BIOLIB_CLOUD_BASE_URL
-            else:
-                if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
-                    cloud_base_url = 'https://biolibcloud.com'
-                else:
-                    raise BioLibError('Calling apps inside apps is not supported in local compute environment')
-
-            if utils.IS_RUNNING_IN_CLOUD:
-                config = CloudUtils.get_webserver_config()
-                s3_results_bucket_name = config['s3_general_storage_bucket_name']
-                s3_results_base_url = f'https://{s3_results_bucket_name}.s3.amazonaws.com'
-            else:
-                if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
-                    s3_results_base_url = 'https://biolib-cloud-api.s3.amazonaws.com'
-                else:
-                    raise BioLibError("Calling apps inside apps locally is only supported on biolib.com")
+            config = CloudUtils.get_webserver_config()
+            compute_node_uuid = config['compute_node_info']['public_id']
+            compute_node_auth_token = config['compute_node_info']['auth_token']
 
             # TODO: Get access_token from new API class instead
             access_token = BiolibApiClient.get().access_token
             bearer_token = f'Bearer {access_token}' if access_token else ''
 
-            nginx_config = f'''
+            nginx_config = f"""
 events {{
   worker_connections  1024;
 }}
@@ -196,29 +177,19 @@ http {{
         default    "";
     }}
 
-    map $request_method $bearer_token_on_patch {{
-        PATCH      "{bearer_token}";
-        default    "";
-    }}
-
     map $request_method $bearer_token_on_patch_and_get {{
         PATCH      "{bearer_token}";
         GET        "{bearer_token}";
         default    "";
     }}
 
-    map $request_method $bearer_token_on_post_and_get {{
-        POST       "{bearer_token}";
-        GET        "{bearer_token}";
-        default    "";
-    }}
-
     server {{
         listen       80;
-        resolver 127.0.0.11 valid=30s;
+        resolver 127.0.0.11 ipv6=off valid=30s;
+        set $upstream_hostname {upstream_hostname};
 
         location ~* "^/api/jobs/cloud/(?<job_id>[a-z0-9-]{{36}})/status/$" {{
-            proxy_pass               {base_url}/api/jobs/cloud/$job_id/status/;
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$job_id/status/;
             proxy_set_header         authorization $bearer_token_on_get;
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
@@ -226,35 +197,35 @@ http {{
 
         location ~* "^/api/jobs/cloud/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/cloud/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/start_upload/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/start_upload/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/start_upload/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/presigned_upload_url/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/presigned_upload_url/$is_args$args;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/presigned_upload_url/$is_args$args;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/complete_upload/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/complete_upload/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/complete_upload/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
         
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/main_result/$" {{
-            proxy_pass                  {base_url}/api/jobs/$job_id/main_result/;
+            proxy_pass                  https://$upstream_hostname/api/jobs/$job_id/main_result/;
             proxy_set_header            authorization "";
             proxy_set_header            cookie "";
             proxy_pass_request_headers  on;
@@ -262,7 +233,7 @@ http {{
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/;
             proxy_set_header         authorization $bearer_token_on_patch_and_get;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
@@ -271,7 +242,7 @@ http {{
 
         location ~* "^/api/jobs/create_job_with_data/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/create_job_with_data/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/create_job_with_data/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
@@ -280,59 +251,65 @@ http {{
 
         location ~* "^/api/jobs/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /api/lfs/ {{
-            proxy_pass               {base_url}/api/lfs/;
-            proxy_set_header         authorization {'$bearer_token_on_post_and_get' if self._can_push_data_record_for_user else '""'};
+        location ~ "^/api/jobs/{self._job_uuid}/notes/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/{self._job_uuid}/notes/$1;
+            proxy_set_header         authorization "";
+            proxy_set_header         job-auth-token "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         compute-node-uuid "{compute_node_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /api/ {{
-            proxy_pass               {base_url}/api/;
+        location /api/lfs/ {{
+            proxy_pass               https://$upstream_hostname/api/lfs/;
             proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /cloud-proxy/ {{
-            proxy_pass               {cloud_base_url}/cloud-proxy/;
+        location /api/app/ {{
+            proxy_pass               https://$upstream_hostname/api/app/;
             proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /job-storage/ {{
-            proxy_pass               {s3_results_base_url}/job-storage/;
+        location /api/ {{
+            proxy_pass               https://$upstream_hostname/api/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/job-storage/ {{
-            proxy_pass               {cloud_base_url}/proxy/storage/job-storage/;
+            proxy_pass               https://$upstream_hostname/proxy/storage/job-storage/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        {f"""
         location /proxy/storage/lfs/versions/ {{
-            proxy_pass               {cloud_base_url}/proxy/storage/lfs/versions/;
+            proxy_pass               https://$upstream_hostname/proxy/storage/lfs/versions/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
-        """ if self._can_push_data_record_for_user else ''}
 
         location /proxy/cloud/ {{
-            proxy_pass               {cloud_base_url}/proxy/cloud/;
+            proxy_pass               https://$upstream_hostname/proxy/cloud/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
@@ -343,15 +320,15 @@ http {{
         }}
     }}
 }}
-'''
+"""
         else:
-            nginx_config = '''
+            nginx_config = """
 events {}
 error_log /dev/stdout info;
 stream {
-    resolver 127.0.0.11 valid=30s;'''
+    resolver 127.0.0.11 valid=30s;"""
             for idx, upstream_server_port in enumerate(upstream_server_ports):
-                nginx_config += f'''
+                nginx_config += f"""
     map "" $upstream_{idx} {{
         default {upstream_server_name}:{upstream_server_port};
     }}
@@ -364,11 +341,11 @@ stream {
     server {{
         listen          {self._remote_host['ports'][idx]} udp;
         proxy_pass      $upstream_{idx};
-    }}'''
+    }}"""
 
-            nginx_config += '''
+            nginx_config += """
 }
-'''
+"""
 
         nginx_config_bytes = nginx_config.encode()
         tarfile_in_memory = io.BytesIO()

biolib/compute_node/webserver/webserver_types.py

@@ -16,5 +16,4 @@ class WebserverConfig(TypedDict):
     base_url: str
     compute_node_info: ComputeNodeInfo
     is_dev: bool
-    s3_general_storage_bucket_name: str
     shutdown_times: ShutdownTimes

biolib/experiments/experiment.py

@@ -1,30 +1,29 @@
 import time
 from collections import OrderedDict
 
-from biolib.biolib_errors import BioLibError
-from biolib.jobs.types import JobsPaginatedResponse
-from biolib.typing_utils import List, Optional
-
+import biolib._internal.types as _types
 from biolib import api
-from biolib.experiments.types import ExperimentDict
+from biolib.biolib_errors import BioLibError
 from biolib.jobs.job import Job
-from biolib.typing_utils import Dict, Union
-
+from biolib.jobs.types import JobsPaginatedResponse
 from biolib.tables import BioLibTable
+from biolib.typing_utils import Dict, List, Optional, Union
 
 
 class Experiment:
     _BIOLIB_EXPERIMENTS: List['Experiment'] = []
 
     # Columns to print in table when showing Job
-    _table_columns_to_row_map = OrderedDict({
-        'Name': {'key': 'name', 'params': {}},
-        'Job Count': {'key': 'job_count', 'params': {}},
-        'Created At': {'key': 'created_at', 'params': {}}
-    })
+    _table_columns_to_row_map = OrderedDict(
+        {
+            'Name': {'key': 'name', 'params': {}},
+            'Job Count': {'key': 'job_count', 'params': {}},
+            'Created At': {'key': 'created_at', 'params': {}},
+        }
+    )
 
-    def __init__(self, name: str):
-        self._experiment_dict: ExperimentDict = self._create_in_backend_or_get_experiment_dict(name)
+    def __init__(self, uri: str, _resource_dict: Optional[_types.ResourceDict] = None):
+        self._resource_dict: _types.ResourceDict = _resource_dict or self._get_or_create_resource_dict(uri)
 
     def __enter__(self):
         Experiment._BIOLIB_EXPERIMENTS.append(self)
@@ -33,18 +32,29 @@ class Experiment:
         Experiment._BIOLIB_EXPERIMENTS.pop()
 
     def __str__(self):
-        return f'Experiment: {self.name}'
+        return f'Experiment: {self.uri}'
 
     def __repr__(self):
-        return f'Experiment: {self.name}'
+        return f'Experiment: {self.uri}'
 
     @property
     def uuid(self) -> str:
-        return self._experiment_dict['uuid']
+        return self._resource_dict['uuid']
 
     @property
     def name(self) -> str:
-        return self._experiment_dict['name']
+        return self._resource_dict['name']
+
+    @property
+    def uri(self) -> str:
+        return self._resource_dict['uri']
+
+    @property
+    def _experiment_dict(self) -> _types.ExperimentSlimDict:
+        if not self._resource_dict['experiment']:
+            raise ValueError(f'Resource {self.uri} is not an Experiment')
+
+        return self._resource_dict['experiment']
 
     @staticmethod
     def get_experiment_in_context() -> Optional['Experiment']:
@@ -55,32 +65,46 @@ class Experiment:
     # Prints a table listing info about experiments accessible to the user
     @staticmethod
     def show_experiments(count: int = 25) -> None:
-        experiment_dicts = api.client.get(
-            path='/experiments/',
-            params={
-                'page_size': str(count)
-            }
-        ).json()['results']
+        experiment_dicts = api.client.get(path='/experiments/', params={'page_size': str(count)}).json()['results']
         BioLibTable(
             columns_to_row_map=Experiment._table_columns_to_row_map,
             rows=experiment_dicts,
-            title='Experiments'
+            title='Experiments',
         ).print_table()
 
+    @staticmethod
+    def get_by_uri(uri: str) -> 'Experiment':
+        query_param_key = 'uri' if '/' in uri else 'name'
+        resource_dict: _types.ResourceDict = api.client.get('/resource/', params={query_param_key: uri}).json()
+        if not resource_dict['experiment']:
+            raise ValueError(f'Resource {uri} is not an experiment')
+
+        return Experiment(uri=resource_dict['uri'], _resource_dict=resource_dict)
+
     def wait(self) -> None:
-        self._refetch_experiment_dict()
+        self._refetch()
         while self._experiment_dict['job_running_count'] > 0:
             print(f"Waiting for {self._experiment_dict['job_running_count']} jobs to finish", end='\r')
             time.sleep(5)
-            self._refetch_experiment_dict()
+            self._refetch()
 
         print(f'All jobs of experiment {self.name} have finished')
 
     def add_job(self, job_id: str) -> None:
-        api.client.patch(
-            path=f'/jobs/{job_id}/',
-            data={'experiment_uuid': self.uuid}
-        )
+        api.client.patch(path=f'/jobs/{job_id}/', data={'experiment_uuid': self.uuid})
+
+    def mount_files(self, mount_path: str) -> None:
+        try:
+            # Only attempt to import FUSE dependencies when strictly necessary
+            from biolib._internal.fuse_mount import (  # pylint: disable=import-outside-toplevel
+                ExperimentFuseMount as _ExperimentFuseMount,
+            )
+        except ImportError as error:
+            raise ImportError(
+                'Failed to import FUSE mounting utils. Please ensure FUSE is installed on your system.'
+            ) from error
+
+        _ExperimentFuseMount.mount_experiment(experiment=self, mount_path=mount_path)
 
     def export_job_list(self, export_format='dicts'):
         valid_formats = ('dicts', 'dataframe')
@@ -98,7 +122,7 @@ class Experiment:
                 raise ImportError(
                     'Pandas must be installed to use this method. '
                     'Alternatively, use .get_jobs() to get a list of job objects.'
-                 ) from error
+                ) from error
 
             jobs_df = pd.DataFrame.from_dict(job_dict_list)
             jobs_df.started_at = pd.to_datetime(jobs_df.started_at)
@@ -125,7 +149,7 @@ class Experiment:
         BioLibTable(
             columns_to_row_map=Job.table_columns_to_row_map,
             rows=[job._job_dict for job in jobs],  # pylint: disable=protected-access
-            title=f'Jobs in experiment: "{self.name}"'
+            title=f'Jobs in experiment: "{self.name}"',
         ).print_table()
 
     def get_jobs(self, status: Optional[str] = None) -> List[Job]:
@@ -147,15 +171,22 @@ class Experiment:
 
         return jobs
 
-    def _create_in_backend_or_get_experiment_dict(self, name: str) -> ExperimentDict:
-        # This endpoint returns experiment dict if already created
-        experiment_dict: ExperimentDict = api.client.post(
-            path='/experiments/',
-            data={
-                'name': name
-            }
-        ).json()
-        return experiment_dict
+    def rename(self, destination: str) -> None:
+        api.client.patch(f'/resources/{self.uuid}/', data={'uri': destination})
+        self._refetch()
+
+    @staticmethod
+    def _get_resource_dict_by_uuid(uuid: str) -> _types.ResourceDict:
+        resource_dict: _types.ResourceDict = api.client.get(f'/resources/{uuid}/').json()
+        if not resource_dict['experiment']:
+            raise ValueError('Resource from URI is not an experiment')
+
+        return resource_dict
+
+    @staticmethod
+    def _get_or_create_resource_dict(uri: str) -> _types.ResourceDict:
+        response_dict = api.client.post(path='/experiments/', data={'uri' if '/' in uri else 'name': uri}).json()
+        return Experiment._get_resource_dict_by_uuid(uuid=response_dict['uuid'])
 
-    def _refetch_experiment_dict(self) -> None:
-        self._experiment_dict = api.client.get(path=f'/experiments/{self.uuid}/').json()
+    def _refetch(self) -> None:
+        self._resource_dict = self._get_resource_dict_by_uuid(uuid=self._resource_dict['uuid'])