cvdlink 0.1.1__py3-none-any.whl

FeatureCloud/api/cli/test/commands.py

@@ -0,0 +1,251 @@
+import os
+import time
+
+import click
+import requests
+
+from FeatureCloud.api.imp.exceptions import FCException
+
+from FeatureCloud.api.imp.test import commands
+from FeatureCloud.api.cli.test.workflow.commands import workflow
+from FeatureCloud.api.imp.test.helper import http
+
+
+@click.group("test")
+def test() -> None:
+    """Testbed related commands"""
+
+
+test.add_command(workflow)
+
+
+@test.command('help')
+def help():
+    _, msg = commands.help()
+    click.echo(msg)
+
+
+@test.command('start')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance (e.g. featurecloud test start --controller-host=http://localhost:8000).')
+@click.option('--client-dirs', default='.,.',
+              help='Client directories separated by comma. The number of clients is based on the number of directories supplied here (e.g. `featurecloud test start --client-dirs=.,.,.,.` command will start 4 clients).',
+              required=True)
+@click.option('--generic-dir', default='.',
+              help='Generic directory available for all clients. Content will be copied to the input folder of all '
+                   'instances (e.g. featurecloud test start --generic-dir=.).',
+              required=True)
+@click.option('--app-image', default='test_app',
+              help='The repository url of the app image (e.g. featurecloud test start --app-image=featurecloud.ai/test_app).',
+              required=True)
+@click.option('--channel', default='local',
+              help='The communication channel to be used. Possible values: "local" or "internet" (e.g. featurecloud test start --channel=local).')
+@click.option('--query-interval', default=2.0,
+              help='The interval after how many seconds the status call will be performed (e.g. featurecloud test start --query-interval=2).')
+@click.option('--download-results',
+              help='A directory name where to download results. This will be created into /data/tests directory (e.g. featurecloud test start --download-results=./results).',
+              default='')
+@click.option('--print-logs',
+              help='When selected, it will monitor the started test by printing the current status every 3s. When the test is finished (or has an error), the relevant logs will be output. Canceling the command after the test was started will NOT stop the test.',
+              is_flag=True)
+def start(controller_host: str, client_dirs: str, generic_dir: str, app_image: str, channel: str, query_interval: str,
+          download_results: str, print_logs: bool):
+    """Starts testbed run with the specified parameters"""
+    try:
+        test_id = commands.start(controller_host, client_dirs, generic_dir, app_image, channel, query_interval,
+                                download_results)
+        click.echo(f"Test id={test_id} started")
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+        return
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+        return
+    except FCException as e:
+        click.echo(f'Error: {e}')
+        return
+    # start monitoring in case wanted (option --print-logs)
+    if print_logs:
+        while True:
+            time.sleep(5)
+            info = commands.info(controller_host, test_id)
+            status = info['status']
+            if len(status) != 1:
+                click.echo('monitoring failed, more than one status of the test found. Please check manually using featurecloud test logs')
+                return
+            status = status.iloc[0].strip()
+            # change format of starttime to be the same than used in controller logs
+            starttime = info['createdAt'].iloc[0].strip()
+            starttime = starttime[:10] + 'T' + starttime[11:]
+            starttime += 'Z'
+            click.echo(f'current Status: {status}')
+            if status in ['error', 'stopped', 'finished']:
+                # controller logs
+                response = http.get(url=f'{controller_host}/logs/?from={0}')
+                controllerString = "CONTROLLER"
+                click.echo(controllerString)
+                click.echo("="*len(controllerString))
+                if response.status_code == 200:
+                    contlogs = response.json()
+                    for logentry in contlogs:
+                        timeentry = logentry['time']
+                        if timeentry >= starttime:
+                            click.echo(logentry)
+                    click.echo('\n\n')
+                else:
+                    click.echo("Could not get controller logs, use the frontend or featurecloud controller logs")
+                # logs of all clients
+                instances = commands.info(controller_host, test_id)['instances']
+                for inst in instances.iloc[0]:
+                    inst_id = inst['id']
+                    logs = commands.logs(controller_host, test_id, inst_id, '0')
+                    clientString = f"CLIENT NUMBER {inst_id}"
+                    click.echo(clientString)
+                    click.echo('='*len(clientString))
+                    click.echo('\n'.join(logs[-50:]) + '\n\n') # only show last 50 lines
+                return
+            elif status in ['running', 'writing']:
+                continue
+            else:
+                click.echo('Unknown status, stopping monitoring. Please check manually using featurecloud test logs or with the frontend')
+                return
+
+
+
+@test.command('stop')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Http address of your running controller instance (e.g. featurecloud test stop --controller-host=http://localhost:8000).',
+              required=True)
+@click.option('--test-id', help='The test id of the test to be stopped. The test id is returned by the start command (e.g.featurecloud test stop --test-id=1).')
+def stop(controller_host: str, test_id: str or int):
+    '''Stops test with specified test id'''
+    try:
+        result = commands.stop(controller_host, test_id)
+        click.echo(f"Test id={result} stopped")
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+@test.command('delete')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance.  (e.g. featurecloud test delete all --controller-host=http://localhost:8000)',)
+@click.option('--test-id', help='The test id of the test to be deleted. The test id is returned by the start command.'
+                                'To delete all tests omit this option and use "delete all".')
+@click.argument('all', type=str, nargs=1, required=False)
+def delete(controller_host: str, test_id: str or int, all: str):
+    '''
+    Deletes test with specified id or alternatively, deletes all tests
+
+     ALL - delete all tests
+
+     Examples:
+
+         featurecloud test delete --test-id=1
+
+         featurecloud test delete all
+    '''
+    try:
+        result = commands.delete(controller_host, test_id, all)
+        if all is not None:
+            if all.lower() == 'all':
+                click.echo(f"All tests deleted")
+            else:
+                click.echo(f'Wrong parameter {all}')
+        else:
+            click.echo(f"Test id={result} deleted")
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+@test.command('list')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance (e.g. featurecloud test list --controller-host=http://localhost:8000).',
+              required=True)
+@click.option('--format', help='Format of the test list. Possible options: json or dataframe (e.g. featurecloud test list --format=dataframe).', required=True, default='dataframe')
+def list(controller_host: str, format: str):
+    '''List all tests'''
+    try:
+        result = commands.list(controller_host, format)
+        if len(result) == 0:
+            click.echo('No tests available')
+        else:
+            click.echo(result)
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+@test.command('info')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance (e.g. featurecloud test info --controller-host=http://localhost:8000).',
+              required=True)
+@click.option('--test-id', help='Test id to get info about (e.g. featurecloud test info --test-id=1).', required=True)
+@click.option('--format', help='Format of the test info. Possible values: json or dataframe (e.g. featurecloud test info --format=dataframe).', required=True, default='dataframe')
+def info(controller_host: str, test_id: str or int, format: str):
+    '''Get information about a running test'''
+    try:
+        result = commands.info(controller_host, test_id, format)
+        click.echo(result)
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+@test.command('traffic')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance (e.g. featurecloud test traffic --controller-host=http://localhost:8000).',
+              required=True)
+@click.option('--test-id', help='The test id to get traffic info about (e.g. featurecloud test traffic --test-id=1).')
+@click.option('--format', help='Format of the test traffic. Possible values: json or dataframe (e.g. featurecloud test traffic --format=dataframe).e', required=True, default='dataframe')
+def traffic(controller_host: str, test_id: str or int, format: str):
+    '''Displays traffic information inside tests'''
+    try:
+        result = commands.traffic(controller_host, test_id, format)
+        click.echo(result)
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+@test.command('logs')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance (e.g. featurecloud test logs --controller-host=http://localhost:8000).',
+              required=True)
+@click.option('--test-id', help='The test id to get logs about (e.g. featurecloud test logs --test-id=1).', required=True)
+@click.option('--instance-id', help='The instance id of the test client. Instance ids can be obtained by running the info command (e.g. featurecloud test logs --test-id=1 --instance-id=0).', required=True)
+@click.option('--from-row', help='Get logs from a certain row number (e.g. featurecloud test logs --test-id=1 --instance-id=0 --from-row=0).', default='', required=True)
+def logs(controller_host: str, test_id: str or int, instance_id: str or int, from_row: str):
+    '''Get logs from test client'''
+    try:
+        result = commands.logs(controller_host, test_id, instance_id, from_row)
+        log_lines = ""
+        for line in result:
+            log_lines += str(line) + os.linesep
+        click.echo(log_lines)
+    except requests.exceptions.InvalidSchema:
+        click.echo(f'No connection adapters were found for {controller_host}')
+    except requests.exceptions.MissingSchema:
+        click.echo(f' Invalid URL {controller_host}: No scheme supplied. Perhaps you meant http://{controller_host}?')
+    except FCException as e:
+        click.echo(f'Error: {e}')
+
+
+if __name__ == "__main__":
+    test()

FeatureCloud/api/cli/test/workflow/commands.py

@@ -0,0 +1,32 @@
+import click
+import importlib
+import sys
+
+
+@click.group("workflow")
+def workflow() -> None:
+    """Workflow related commands"""
+
+
+@workflow.command('start')
+@click.option('--controller-host', default='http://localhost:8000',
+              help='Address of your running controller instance.',
+              required=True)
+@click.option('--wf-dir', default='',
+              help='path to directory containing the workflow',
+              required=True)
+@click.option('--wf-file', default='example_wf',
+              help='python file including the workflow',
+              required=True)
+@click.option('--channel', default='local',
+              help='The communication channel to be used. Can be local or internet.',
+              required=True)
+@click.option('--query-interval', default=1,
+              help='The interval after how many seconds the status call will be performed.',
+              required=True)
+def start_workflow(controller_host: str, wf_dir: str, wf_file: str, channel: str, query_interval: str):
+    sys.path.append(wf_dir)
+    workflow_class = importlib.import_module(wf_file)
+    wf = workflow_class.WorkFlow(controller_host=controller_host, channel=channel, query_interval=query_interval)
+    wf.register_apps()
+    wf.run()

FeatureCloud/api/imp/app/commands.py

@@ -0,0 +1,278 @@
+import git
+import os
+from urllib.parse import urljoin
+import docker
+from docker import DockerClient
+import sys
+from git import GitError
+
+from FeatureCloud.api.imp.exceptions import FCException
+from FeatureCloud.api.imp.util import getcwd_fslash, get_docker_client, remove_dir
+import pydot
+import importlib
+
+FC_REPO_PREFIX = "featurecloud.ai/"
+
+
+def create_link(template_name: str) -> str:
+    TEMPLATE_URL = 'https://github.com/FeatureCloud/'
+    return urljoin(TEMPLATE_URL, template_name + '.git')
+
+
+def new(name: str, directory: str = '.', template_name: str = 'app-blank') -> str:
+    """
+    Parameters
+    ----------
+    name: str
+        app name
+    directory: str
+        path to create the app's directory in
+    template_name: str
+        a template repository in FeatureCloud repositories
+    Returns
+    -------
+    str
+        path to the created app
+    Raises
+    -------
+         FeatureCloud.api.imp.exceptions.FCException
+    """
+
+    if not template_name.startswith('app-'):
+        raise FCException(f'{template_name} is not a valid template. Please see the help for available templates')
+
+    try:
+        app_path = os.path.join(directory, name.lower())
+        # depth=1 clones with minimal history information, we delete the .git folder anyway
+        repo = git.Repo.clone_from(create_link(template_name), app_path, multi_options=['--depth=1'])
+        repo.delete_remote('origin')
+    except GitError as e:
+        raise FCException(e)
+
+    # delete the .git folder
+    try:
+        remove_dir(os.path.join(app_path, '.git'))
+    except:
+        pass
+
+    return app_path
+
+
+def build(path: str = ".", image_name: str = None, tag: str = "latest", rm: str = True):
+    """ Build app image.
+
+    Parameters
+    ----------
+    path: str
+        path to the directory containing the Dockerfile
+    image_name: str
+        name of the app's docker image. If not provided, the name of the current directory will be used.
+    tag: str
+        versioning tag
+    rm: bool
+        if True, remove intermediate containers
+    Returns
+    -------
+    lines
+       generator providing information about the build progress
+    Raises
+    -------
+         FeatureCloud.api.imp.exceptions.FCException
+    """
+    if image_name is None:
+        image_name = getcwd_fslash().split("/")[-1]
+
+    client = get_docker_client()
+    try:
+        # Docker would also check for Dockerfile, but if, by accident, a path is provided with many sub folders
+        # it just takes too long. So we check it upfront:
+        if not os.path.exists(os.path.join(path, 'Dockerfile')):
+            raise FCException(f'Dockerfile not found in directory: {os.path.abspath(path)}')
+        build_log_generator = client.images.build(path=path, tag=f"{image_name.lower()}:{tag}", rm=rm)
+        if type(build_log_generator) == tuple:
+            return build_log_generator[1]
+        else:
+            raise Exception("Unexpected response from the docker SDK")
+    except docker.errors.BuildError as e:
+        raise FCException("Error in building the image: " + str(e))
+    except docker.errors.APIError as e:
+        raise FCException("Error with the API: " + str(e))
+    except TypeError as e:
+        raise FCException("Error path to the Dockerfile not given: " + str(e))
+
+
+def download(name: str, tag: str = "latest"):
+    """ Download a given docker image from FeatureCloud.ai docker repo.
+
+    Parameters
+    ----------
+    name: str
+        image name
+    tag: str
+        versioning tag
+    Returns
+    -------
+    lines
+       generator providing information about the download progress
+    Raises
+    -------
+         FeatureCloud.api.imp.exceptions.FCException
+    """
+    fc_name = fc_repo_name(name)
+    client = get_docker_client()
+    try:
+        for entry in client.api.pull(repository=fc_name, tag=tag, stream=True, decode=True):
+            if 'error' in entry:
+                raise FCException(entry['error'])
+
+            yield entry
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+
+def publish(name: str, tag: str = "latest"):
+    """ Push a given app into FeatureCloud.ai docker repo.
+
+    Parameters
+    ----------
+    name: str
+        image name
+    tag: str
+        versioning tag
+    Returns
+    -------
+    lines
+       generator providing information about the publish progress
+    Raises
+    -------
+         FeatureCloud.api.imp.exceptions.FCException
+    """
+    fc_name = fc_repo_name(name)
+    client = get_docker_client()
+
+    try:
+        local_repo = f'{name}:{tag}'
+        registry_repo = f'{fc_name}:{tag}'
+        client.images.get(local_repo).tag(registry_repo)
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+    try:
+        for entry in client.api.push(repository=registry_repo, stream=True, decode=True):
+            # Examples of stream entries:
+            # {'status': 'Waiting', 'progressDetail': {}, 'id': '38feb0548c7a'}
+            # {'errorDetail': {'message': 'unauthorized: authentication required'}, 'error': 'unauthorized: authentication required'}
+            if 'error' in entry:
+                raise FCException(entry['error'])
+
+            yield entry
+    except docker.errors.DockerException as e:
+        raise FCException(e)
+
+
+def remove(name: str, tag: str = "latest"):
+    """ Delete docker image from local hard drive.
+
+    Parameters
+    ----------
+    name: str
+        image name
+    tag: str
+        the versioning tag to be removed. If set to 'all', all versions will be deleted.
+    Raises
+    -------
+         FeatureCloud.api.imp.exceptions.FCException
+    """
+    to_find = name if tag == 'all' else f'{name}:{tag}'
+    client = get_docker_client()
+    removed = remove_images(client, to_find)
+
+    if tag == 'all':
+        # search also for local images containing the FC repository name
+        to_find = fc_repo_name(name)
+        removed = removed + remove_images(client, to_find)
+
+    return removed
+
+
+def fc_repo_name(name: str) -> str:
+    if not name.startswith(FC_REPO_PREFIX):
+        return f'{FC_REPO_PREFIX}{name}'
+
+    return name
+
+
+def remove_images(client: DockerClient, to_find: str):
+    removed = []
+    for img in client.images.list(to_find):
+        # one image id may have several tags
+        for img_tag in img.tags:
+            try:
+                client.images.remove(image=img_tag, force=True)
+            except docker.errors.DockerException as e:
+                raise FCException(e)
+        removed = removed + img.tags
+
+    return removed
+
+
+def plot_state_diagram(path: str, package: str, states: str, plot_name: str):
+    states = list(filter(None, states.split(',')))
+    sys.path.append(path)
+    if "/" in package:
+        sub_pkg_path = path
+        for sub_pkg in package.split("/"):
+            sub_pkg_path += "/" + sub_pkg
+            sys.path.append(sub_pkg_path)
+    else:
+        sys.path.append(f"{path}/{package}")
+    if states == 'main':
+        from FeatureCloud.app.engine.app import app
+        main_py = load_module(states[0], f"{path}/{package}/{states[0]}.py")
+        app = main_py.app
+
+    else:
+        for state in states:
+            load_module(state, f"{path}/{package}/{state}.py")
+        from FeatureCloud.app.engine.app import app
+
+
+    app.register()
+
+    graph = pydot.Dot('FeatureCloud State Diagram', graph_type='digraph', bgcolor='transparent')
+
+    for s in app.states:
+        state = app.states[s]
+        state_node = pydot.Node(state.name, label=state.name)
+        if state.coordinator and state.participant:
+            state_node.set('color', 'purple')
+        elif state.coordinator:
+            state_node.set('color', 'red')
+        elif state.participant:
+            state_node.set('color', 'blue')
+        if state.name == 'initial' or state.name == 'terminal':
+            state_node.set('peripheries', 2)
+        graph.add_node(state_node)
+
+    for t in app.transitions:
+        transition = app.transitions[t]
+        label = t if transition[4] is None else transition[4]
+
+        state_edge = pydot.Edge(transition[0].name, transition[1].name, label=label)
+        if transition[2] and transition[3]:
+            state_edge.set('color', 'purple')
+        elif transition[3]:
+            state_edge.set('color', 'red')
+        elif transition[2]:
+            state_edge.set('color', 'blue')
+        graph.add_edge(state_edge)
+
+    graph.write_png(f"{path}/{plot_name}.png")
+
+
+def load_module(module, path):
+    loader = importlib.machinery.SourceFileLoader(module, path)
+    spec = importlib.util.spec_from_loader(module, loader)
+    mymodule = importlib.util.module_from_spec(spec)
+    loader.exec_module(mymodule)
+    return mymodule