cvdlink 0.1.1__py3-none-any.whl

FeatureCloud/api/imp/controller/commands.py

@@ -0,0 +1,246 @@
+import tqdm
+import time
+import docker
+import os
+
+from FeatureCloud.api.imp.exceptions import FCException, ContainerNotFound
+from FeatureCloud.api.imp.test.helper import http
+
+from FeatureCloud.api.imp.util import getcwd_fslash, get_docker_client
+
+DEFAULT_CONTROLLER_IMAGE = "featurecloud.ai/controller"
+CONTROLLER_LABEL = "FCControllerLabel"
+DEFAULT_PORT = 8000
+DEFAULT_CONTROLLER_NAME = 'fc-controller'
+DEFAULT_DATA_DIR = 'data'
+LOG_FETCH_INTERVAL = 3  # seconds
+LOG_LEVEL_CHOICES = ['debug', 'info', 'warn', 'error', 'fatal']
+
+
+def start(
+    name: str,
+    port: int,
+    data_dir: str,
+    controller_image: str,
+    with_gpu: bool,
+    mount: str,
+    blockchain_address: str,
+    profile: str = None,
+    global_endpoint: str = "",
+    registry: str = "",
+    relay_address: str = "",
+    poll_interval: int = 0,
+    query_interval: int = 0,
+    config_file: str = "",
+):
+    client = get_docker_client()
+
+    data_dir = data_dir if data_dir else DEFAULT_DATA_DIR
+    if not controller_image:
+        if profile == "cvdlink":
+            controller_image = "fc.cvdlink-project.eu/controller:latest"
+        else:
+            controller_image = DEFAULT_CONTROLLER_IMAGE
+    # Create data dir if needed
+    try:
+        os.mkdir(data_dir)
+    except OSError as error:
+        pass
+
+    # cleanup unused controller containers and networks
+    prune_controllers()
+
+    # remove controller having the same name, if any
+    try:
+        client.api.remove_container(name, v=True, force=True)
+    except docker.errors.NotFound:
+        pass
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+    # pull controller and display progress
+    try:
+        pull_proc = client.api.pull(repository=controller_image, stream=True)
+        for p in tqdm.tqdm(pull_proc, desc='Downloading...'):
+            pass
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+    cont_name = name if name else DEFAULT_CONTROLLER_NAME
+    # forward slash works on all platforms
+    base_dir = getcwd_fslash()
+
+    # set the correct host folder to be mounted
+    if os.path.isabs(data_dir):
+        host_data_dir = data_dir
+    else:
+        host_data_dir = os.path.join(base_dir, data_dir)
+
+    volumes = [f'{host_data_dir}:/{data_dir}', '/var/run/docker.sock:/var/run/docker.sock']
+    if mount:
+        volumes.append(f'{mount}:/mnt')
+
+    # ---------- profile-based defaults ----------
+    # For featurecloud: use config.yml defaults unless user overrides via CLI.
+    # For cvdlink: provide sensible defaults if not explicitly set.
+    if profile == "cvdlink":
+        if not global_endpoint:
+            global_endpoint = "https://fc.cvdlink-project.eu/api/"
+        if not registry:
+            registry = "fc.cvdlink-project.eu"
+        if not relay_address:
+            relay_address = "relay.fc.cvdlink-project.eu"
+
+
+    # ---------- build controller command ----------
+    # NB: we keep the original style with a single command string.
+    cmd_parts = [
+        f"--host-root='{host_data_dir}'",
+        f"--internal-root=/{data_dir}",
+        f"--controller-name={cont_name}",
+    ]
+    device_requests = None
+    if with_gpu:
+        device_requests = [docker.types.DeviceRequest(count=-1, capabilities=[['gpu']])]
+        cmd_parts.append("--has-gpu")
+
+    if blockchain_address:
+        cmd_parts.append(f"--blockchain-address={blockchain_address}")
+
+    if config_file:
+        cmd_parts.append(f"--configfile={config_file}")
+
+    if global_endpoint:
+        cmd_parts.append(f"--endpoint='{global_endpoint}'")
+
+    if registry:
+        cmd_parts.append(f"--registry={registry}")
+
+    if relay_address:
+        cmd_parts.append(f"--address={relay_address}")
+
+    if poll_interval > 0:
+        cmd_parts.append(f"--poll-interval={poll_interval}")
+
+    if query_interval > 0:
+        cmd_parts.append(f"--query-interval={query_interval}")
+
+    command = " ".join(cmd_parts)
+
+    try:
+        client.containers.run(
+            controller_image,
+            detach=True,
+            name=cont_name,
+            ports={8000: port if port else DEFAULT_PORT},
+            volumes=volumes,
+            labels=[CONTROLLER_LABEL],
+            device_requests=device_requests,
+            command=command,
+        )
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+    except Exception as e:
+        raise FCException(e)
+
+
+def stop(name: str):
+    client = get_docker_client()
+
+    if not name:
+        name = DEFAULT_CONTROLLER_NAME
+
+    # Removing controllers filtered by name
+    removed = []
+    for container in client.containers.list(filters={"name": [name]}):
+        try:
+            client.api.remove_container(container.id, v=True, force=True)
+            removed.append(container.name)
+        except docker.errors.DockerException as e:
+            raise FCException(e)
+
+    return removed
+
+
+def prune_controllers():
+    client = get_docker_client()
+
+    try:
+        client.containers.prune(filters={"label": [CONTROLLER_LABEL]})
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+    # Remove the network
+    try:
+        # Attempt to remove the network
+        client.networks.get("fc-controller-internal").remove()
+    except docker.errors.NotFound:
+        pass
+    except docker.errors.APIError as e:
+        print(f"An API error occurred: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+
+def logs(name: str, tail: bool, log_level: str):
+    client = get_docker_client()
+
+    # get controller address
+    host_port = 0
+    try:
+        container = client.containers.get(name)
+        host_port = container.attrs['NetworkSettings']['Ports']['8000/tcp'][0]['HostPort']
+    except docker.errors.NotFound:
+        raise ContainerNotFound(name)
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+    # Get logs content from controller
+    lines_fetched = 0
+    while True:
+        response = http.get(url=f'http://localhost:{host_port}/logs/?from={lines_fetched}')
+
+        if response.status_code == 200:
+            logs_content = response.json()
+            lines_fetched += len(logs_content)
+
+            log_level = valid_log_level(log_level)
+            filtered = filter_logs(logs_content, log_level)
+            for line in filtered:
+                yield line
+        else:
+            raise FCException(response.json('detail'))
+
+        if not tail:
+            break
+        time.sleep(LOG_FETCH_INTERVAL)
+
+
+def status(name: str):
+    client = get_docker_client()
+    try:
+        return client.containers.get(name)
+    except docker.errors.NotFound:
+        raise ContainerNotFound(name)
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+
+def ls():
+    client = get_docker_client()
+    try:
+        return client.containers.list(filters={'label': [CONTROLLER_LABEL]})
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+
+def filter_logs(logs_json, log_level):
+    log_level_index = LOG_LEVEL_CHOICES.index(log_level)
+    return [line for line in logs_json if LOG_LEVEL_CHOICES.index(line['level']) >= log_level_index]
+
+
+def valid_log_level(log_level):
+    if len(log_level) == 0 or log_level not in LOG_LEVEL_CHOICES:
+        log_level = LOG_LEVEL_CHOICES[0]
+
+    return log_level

FeatureCloud/api/imp/exceptions.py

@@ -0,0 +1,29 @@
+
+class FCException(Exception):
+    """
+    Base class for FeatureCloud exceptions.
+    """
+    default_detail = 'an error occurred'
+
+    def __init__(self, detail=None):
+        if detail is None:
+            detail = self.default_detail
+
+        self.detail = detail
+
+    def __str__(self):
+        return str(self.detail)
+
+
+class DockerNotAvailable(FCException):
+    default_detail = 'Docker daemon is not available. Please make sure Docker is started.'
+
+
+class ControllerOffline(FCException):
+    def __init__(self, url=None):
+        super().__init__(f'Could not access controller on URL: {url}')
+
+
+class ContainerNotFound(FCException):
+    def __init__(self, name=None):
+        super().__init__(f'Container not found: {name}')

FeatureCloud/api/imp/test/api/backend/auth.py

@@ -0,0 +1,54 @@
+import json
+from pathlib import Path
+
+from FeatureCloud.api.utils.cli.helper import http
+
+home = str(Path.home())
+url = 'http://127.0.0.1:7000'
+
+
+def is_user_logged_in():
+    headers = create_authorization_header()
+    r = http.get(url=f'{url}/user/info/', headers=headers)
+    if r.status_code == 200:
+        return True
+    else:
+        print("No user is logged in. Please login first.")
+        exit()
+        return False
+
+
+def login(email: str, password: str):
+    r = http.post(url=f'{url}/auth/login/',
+                  json={'username': email,
+                        'password': password})
+    if r.status_code == 200:
+        refresh, access = r.json()['refresh'], r.json()['access']
+        login_details = {'email': email, 'password': password, 'refresh': refresh, 'access': access}
+        login_details = json.dumps(login_details)
+        file = open(f'{home}/.fc_cli', 'w')
+        file.write(login_details)
+        file.close()
+        return True
+    else:
+        return False
+
+
+def logout():
+    r = http.post(url=f'{url}/auth/logout/',
+                  json={'refresh': None})
+    if r.status_code == 200:
+        with open(f'{home}/.fc_cli', 'w') as file:
+            file.write('')
+        file.close()
+        return True
+    else:
+        print(r.content)
+        return False
+
+
+def create_authorization_header():
+    with open(f'{home}/.fc_cli') as jsonFile:
+        access_token = json.load(jsonFile)['access']
+        jsonFile.close()
+    return {"Authorization": f'Bearer {access_token}'}

FeatureCloud/api/imp/test/api/backend/project.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+from FeatureCloud.api.utils.cli.api.backend.auth import create_authorization_header
+from FeatureCloud.api.utils.cli.helper import http
+
+home = str(Path.home())
+url = 'http://127.0.0.1:7000'
+
+
+def info(project_id: str or int):
+    headers = create_authorization_header()
+    r = http.get(url=f'{url}/projects/{project_id}', headers=headers)
+    if r.status_code == 200:
+        return r.json()
+    else:
+        return False
+
+
+def create(name: str, description: str):
+    headers = create_authorization_header()
+    r = http.post(url=f'{url}/projects/',
+                  json={'name': name,
+                        'description': description},
+                  headers=headers
+                  )
+
+    if r.status_code == 200:
+        return r.json()['id']
+    else:
+        return False
+
+
+def remove(proj_id: str or int):
+    headers = create_authorization_header()
+    r = http.delete(url=f'{url}/projects/{proj_id}/', headers=headers)
+
+    if r.status_code == 200:
+        return True
+    else:
+        return False
+
+
+def list_projects():
+    headers = create_authorization_header()
+    r = http.get(url=f'{url}/projects/', headers=headers)
+
+    if r.status_code == 200:
+        return r.json()
+    else:
+        return False
+
+
+def list_tokens(project_id):
+    headers = create_authorization_header()
+    r = http.get(url=f'{url}/project-tokens/{project_id}/', headers=headers)
+    if r.status_code == 200:
+        return r.json()
+    else:
+        return False
+
+
+def create_token(project_id: str or int):
+    headers = create_authorization_header()
+    r = http.post(url=f'{url}/project-tokens/{project_id}/',
+                  json={'cmd': 'create'},
+                  headers=headers
+                  )
+
+    if r.status_code == 200:
+        return r.json()
+    else:
+        return False
+
+
+def remove_token(token_id: str or int):
+    headers = create_authorization_header()
+    r = http.delete(url=f'{url}/project-tokens/token/{token_id}/', headers=headers)
+
+    if r.status_code == 200:
+        return r.json()
+    else:
+        print(r.status_code)
+        print(r.text)
+        return False

FeatureCloud/api/imp/test/api/controller.py

@@ -0,0 +1,97 @@
+import requests
+from urllib.parse import urlencode
+
+from FeatureCloud.api.imp.test.helper import http
+
+
+def is_online(url: str = 'http://localhost:8000/'):
+    try:
+        requests.get(url=url)
+        return True
+    except requests.exceptions.ConnectionError:
+        return False
+
+
+def start_test(url: str, docker_image: str, directories: [], generic_directory: str, is_local_relay: bool,
+               query_interval: int, download_results: str):
+    """
+    Start a new test
+    :param url:
+    :param docker_image:
+    :param directories:
+    :param generic_directory
+    :param is_local_relay:
+    :param query_interval:
+    :param download_results:
+    :return:
+    """
+
+    response = http.post(url=f'{url}/app/test/', json={'dockerImage': docker_image,
+                                                       'directories': list(directories),
+                                                       'genericDirectory': generic_directory,
+                                                       'isLocalRelay': is_local_relay,
+                                                       'queryInterval': query_interval,
+                                                       'downloadResults': download_results})
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json()
+
+
+def delete_test(url: str, test_id: str or int):
+    response = http.delete(url=f'{url}/app/test/{test_id}/?delete=true')
+    if response.status_code == 200:
+        return True, response
+    else:
+        return False, response.json()
+
+
+def stop_test(url: str, test_id: str or int):
+    response = http.delete(url=f'{url}/app/test/{test_id}/?delete=false')
+    if response.status_code == 200:
+        return True, response.content
+    else:
+        return False, response.json()
+
+
+def get_tests(url: str):
+    response = http.get(url=f'{url}/app/tests/')
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json()
+
+
+def delete_tests(url: str):
+    response = http.delete(url=f'{url}/app/tests/')
+    if response.status_code == 200:
+        return True, response.content
+    else:
+        return False, response.json()
+
+
+def get_test(url: str, test_id: str or int):
+    response = http.get(url=f'{url}/app/test/{test_id}/')
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json()
+
+
+def get_traffic(url: str, test_id: str or int):
+    response = http.get(url=f'{url}/app/test/{test_id}/traffic/')
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json()
+
+
+def get_logs(url: str, test_id: str or int, instance_id: str or int, from_param: str):
+    response = http.get(url=f'{url}/app/test/{test_id}/instance/{instance_id}/logs/?from={from_param}')
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json()

FeatureCloud/api/imp/test/commands.py

@@ -0,0 +1,124 @@
+from FeatureCloud.api.imp.test import helper
+from FeatureCloud.api.imp.test.api import controller
+from FeatureCloud.api.imp.exceptions import ControllerOffline, FCException
+
+
+def help():
+    return (None, """For registering and testing your apps or using other apps, please visit 
+          our 
+          website: \n https://featurecloud.ai.\n And for more information about
+           FeatureCloud architecture: \n
+          The FeatureCloud AI Store for Federated Learning in Biomedicine and 
+          Beyond\n 
+          https://arxiv.org/abs/2105.05734 """)
+
+
+def start(controller_host: str, client_dirs: str, generic_dir: str, app_image: str, channel: str, query_interval: int,
+          download_results: str):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.start_test(controller_host,
+                                            app_image,
+                                            filter(None, client_dirs.split(',')),
+                                            generic_dir,
+                                            channel == 'local',
+                                            query_interval,
+                                            download_results)
+
+    if success:
+        return result['id']
+    else:
+        raise FCException(result['detail'])
+
+
+def stop(controller_host: str, test_id: str or int):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.stop_test(controller_host, test_id)
+
+    if success:
+        return test_id
+    else:
+        raise FCException(result['detail'])
+
+
+def delete(controller_host: str, test_id: str or int, del_all: str):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    if test_id is not None and del_all is None:
+        success, result = controller.delete_test(controller_host, test_id)
+
+        if success:
+            return test_id
+        else:
+            raise FCException(result['detail'])
+
+    elif test_id is None and len(del_all) > 0:
+        if del_all.lower() == 'all':
+            success, result = controller.delete_tests(controller_host)
+
+            if success:
+                return 'all'
+            else:
+                raise FCException(result['detail'])
+        else:
+            raise FCException(f'Unsupported argument {del_all}')
+
+    else:
+        raise FCException('Wrong combination of parameters. To delete a single test use option --test-id. To delete all tests use the "all" argument.')
+
+
+def list(controller_host: str, format: str = 'dataframe'):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.get_tests(controller_host)
+    if success:
+        if format == 'json':
+            return result
+        else:
+            return helper.json_to_dataframe(result).set_index('id')
+    else:
+        raise FCException(result)
+
+
+def info(controller_host: str, test_id: str or int, format: str = 'dataframe'):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.get_test(controller_host, test_id)
+    if success:
+        if format == 'json':
+            return result
+        else:
+            return helper.json_to_dataframe(result, single_entry=True).set_index('id')
+    else:
+        raise FCException(result['detail'])
+
+
+def traffic(controller_host: str, test_id: str or int, format: str):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.get_traffic(controller_host, test_id)
+    if success:
+        if format == 'json':
+            return result
+        else:
+            return helper.json_to_dataframe(result)
+    else:
+        raise FCException(result['detail'])
+
+
+def logs(controller_host: str, test_id: str or int, instance_id: str or int, from_param: str):
+    if not controller.is_online(controller_host):
+        raise ControllerOffline(controller_host)
+
+    success, result = controller.get_logs(controller_host, test_id, instance_id, from_param)
+    if success:
+        return result["logs"]
+    else:
+        raise FCException(result['detail'])

FeatureCloud/api/imp/test/helper.py

@@ -0,0 +1,40 @@
+import pandas as pd
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+retry_strategy = Retry(
+    total=3,
+    status_forcelist=[429, 500, 502, 503, 504],
+    method_whitelist=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"],
+    backoff_factor=1
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+http = requests.Session()
+http.mount("https://", adapter)
+http.mount("http://", adapter)
+
+
+def json_to_dataframe(json, single_entry: bool = False, default_column: str = 'id'):
+    if len(json) == 0:
+        # return empty data frame
+        df = pd.DataFrame({default_column: []})
+    elif not single_entry:
+        df = pd.DataFrame.from_records(json)
+    else:
+        df = pd.DataFrame.from_dict(json, orient='index').T
+
+    try:
+        df.createdAt = pd.to_datetime(df.createdAt).apply(lambda x: str(x).split(".")[0])
+    except Exception:
+        pass
+    try:
+        df.finishedAt = pd.to_datetime(df.finishedAt).apply(lambda x: str(x).split(".")[0])
+    except Exception:
+        pass
+    pd.set_option('display.max_rows', None)
+    pd.set_option('display.max_columns', None)
+    pd.set_option('display.width', None)
+    pd.set_option('display.max_colwidth', None)
+
+    return df