cgcsdk 1.0.6__py3-none-any.whl → 1.0.9__py3-none-any.whl

cgc/.env

@@ -6,4 +6,4 @@ CONFIG_FILE_NAME = cfg.json
 TMP_DIR = .tmp
 RELEASE = 1
 MAJOR_VERSION = 0
-MINOR_VERSION = 6
+MINOR_VERSION = 9

cgc/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Change Log
 
+## 1.0.9
+
+Release on April 23, 2024
+
+* updated cgc to work with cgc-server 1.1.0
+* updated registration process
+* updated available apps, so it reflects changes in cgc-server
+* hotfix: listening jobs, which are not Running
+
+## 1.0.8
+
+Release on April 11, 2024
+
+* added new command group: cgc keys
+  * ssh create - create new ssh key, for ssh access
+  * ssh list - list all ssh keys
+  * ssh delete - delete ssh key
+  * ssh update - update ssh key
+
+## 1.0.7
+
+Release on April 04, 2024
+
+* speed of resource creation with volumes / shared memory, has been improved
+* added jobs creation: cgc job create
+* added jobs list: cgc job list
+* added jobs delete: cgc job delete
+* added logs (STDOUT, STDERR) reader for spawned applications: cgc logs
+* added SDK for logs & jobs
+
 ## 1.0.6
 
 Release on March 13, 2024

cgc/cgc.py

@@ -4,6 +4,7 @@ from cgc.commands.compute.compute_cmd import compute_group
 from cgc.commands.billing.billing_cmd import billing_group
 from cgc.commands.db.db_cmd import db_group
 from cgc.commands.resource.resource_cmd import resource_group
+from cgc.commands.jobs.jobs_cmd import job_group
 from cgc.commands.auth.auth_cmd import (
     api_keys_group,
     auth_register,
@@ -13,10 +14,13 @@ from cgc.commands.debug.debug_cmd import debug_group
 from cgc.commands.cgc_cmd import (
     cgc_rm,
     cgc_status,
+    cgc_logs,
     sending_telemetry_permission,
     resource_events,
     context_group,
 )
+from cgc.commands.keys.keys_cmd import keys_group
+
 from cgc.utils.version_control import check_version, _get_version
 from cgc.utils.click_group import CustomGroup
 
@@ -41,7 +45,9 @@ cli.add_command(resource_group)
 cli.add_command(billing_group)
 cli.add_command(cgc_status)
 cli.add_command(sending_telemetry_permission)
-
+cli.add_command(cgc_logs)
+cli.add_command(job_group)
+cli.add_command(keys_group)
 
 if __name__ == "__main__" or __name__ == "cgc.cgc":
     cli()

cgc/commands/auth/__init__.py

@@ -7,9 +7,9 @@ class AuthCommandException(ResponseException):
 
 class NoNamespaceInConfig(AuthCommandException):
     def __init__(self) -> None:
-        super().__init__(f"Namespace not readable from config file.")
+        super().__init__("Namespace not readable from config file.")
 
 
 class NoConfigFileFound(AuthCommandException):
     def __init__(self) -> None:
-        super().__init__(f"Config does not exists.")
+        super().__init__("Config does not exists.")

cgc/commands/auth/auth_cmd.py

@@ -9,7 +9,11 @@ from cgc.commands.auth.auth_utils import (
     auth_delete_api_key,
     auth_list_api_keys,
 )
-from cgc.utils.prepare_headers import get_url_and_prepare_headers_register
+from cgc.utils.prepare_headers import (
+    get_url_and_prepare_headers_register,
+    load_user_api_url,
+    load_user_cgc_secret,
+)
 from cgc.utils.cryptography import rsa_crypto
 from cgc.utils.click_group import CustomCommand, CustomGroup
 from cgc.utils.requests_helper import call_api, EndpointTypes
@@ -20,6 +24,7 @@ from cgc.utils.response_utils import (
 )
 from cgc.utils import (
     check_if_config_exist,
+    require_answer_loop,
     require_confirm_loop,
     find_first_available_config_name,
 )
@@ -55,9 +60,14 @@ def auth_register(config_filename: str = "cfg.json"):
         require_confirm_loop("Do you want to add new context?")
         config_filename = find_first_available_config_name()
 
+    cgc_api_url = require_answer_loop("Enter CGC server address", load_user_api_url())
+    cgc_secret = require_answer_loop("Enter CGC secret", load_user_cgc_secret())
+
     user_id = input("User ID: ")
     access_key = input("Access key: ")
-    url, headers = get_url_and_prepare_headers_register(user_id, access_key)
+    url, headers = get_url_and_prepare_headers_register(
+        user_id, access_key, cgc_api_url, cgc_secret
+    )
     metric = "auth.register"
     pub_key_bytes, priv_key_bytes = rsa_crypto.key_generate_pair()
     __payload = pub_key_bytes
@@ -74,6 +84,8 @@ def auth_register(config_filename: str = "cfg.json"):
             user_id,
             priv_key_bytes,
             config_filename,
+            cgc_api_url,
+            cgc_secret,
         )
     )
 

cgc/commands/auth/auth_responses.py

@@ -8,14 +8,21 @@ from cgc.utils.message_utils import key_error_decorator_for_helpers
 
 
 @key_error_decorator_for_helpers
-def auth_register_response(response, user_id, priv_key_bytes, config_filename) -> str:
+def auth_register_response(
+    response, user_id, priv_key_bytes, config_filename, cgc_api_url, cgc_secret
+) -> str:
     TMP_DIR_PATH = os.path.join(get_config_path(), TMP_DIR)
     unzip_dir, namespace = auth_utils.save_and_unzip_file(response)
     aes_key, password = auth_utils.get_aes_key_and_password(unzip_dir, priv_key_bytes)
 
     os.environ["CONFIG_FILE_NAME"] = config_filename
     save_to_config(
-        user_id=user_id, password=password, aes_key=aes_key, namespace=namespace
+        user_id=user_id,
+        password=password,
+        aes_key=aes_key,
+        namespace=namespace,
+        cgc_api_url=cgc_api_url,
+        cgc_secret=cgc_secret,
     )
     auth_utils.auth_create_api_key_with_save()
     shutil.rmtree(TMP_DIR_PATH)

cgc/commands/auth/auth_utils.py

@@ -10,7 +10,7 @@ from cgc.utils.config_utils import save_to_config
 from cgc.utils.config_utils import read_from_cfg
 from cgc.utils.cryptography import rsa_crypto
 from cgc.utils import prepare_headers
-from cgc.utils.consts.env_consts import API_URL, TMP_DIR
+from cgc.utils.consts.env_consts import TMP_DIR
 
 from cgc.utils.requests_helper import call_api, EndpointTypes
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
@@ -29,9 +29,10 @@ def _get_jwt_from_server(user_id: str = None, password: str = None) -> str:
     if user_id is None or password is None:
         if user_id is not None or password is not None:
             raise ValueError("Both user_id and password must be provided")
+    if user_id is None and password is None:
         user_id = urllib.parse.quote(read_from_cfg("user_id"))
         password = urllib.parse.quote(read_from_cfg("password"))
-    url = f"{API_URL}/v1/api/user/create/token"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/create/token"
     headers = {
         "accept": "application/json",
         "Content-Type": "application/x-www-form-urlencoded",
@@ -54,7 +55,7 @@ def get_jwt(user_id: str = None, password: str = None) -> str:
 
 def auth_create_api_key_with_save(user_id: str = None, password: str = None):
     """Function to create API key and API secret for user and save it to config file."""
-    url = f"{API_URL}/v1/api/user/create/api-key"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/create/api-key"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(
@@ -74,7 +75,7 @@ def auth_create_api_key_with_save(user_id: str = None, password: str = None):
 
 def auth_delete_api_key(api_key: str, user_id: str = None, password: str = None):
     """Function to delete API key."""
-    url = f"{API_URL}/v1/api/user/delete/api-key?api_key={api_key}"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/delete/api-key?api_key={api_key}"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(
@@ -89,7 +90,7 @@ def auth_delete_api_key(api_key: str, user_id: str = None, password: str = None)
 
 def auth_list_api_keys(user_id: str = None, password: str = None):
     """Function to list API keys."""
-    url = f"{API_URL}/v1/api/user/list/api-key"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/list/api-key"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(

cgc/commands/cgc_cmd.py

@@ -1,7 +1,10 @@
 import click
 import json
 
-from cgc.commands.cgc_cmd_responses import cgc_status_response
+from cgc.commands.cgc_cmd_responses import (
+    cgc_logs_response,
+    cgc_status_response,
+)
 from cgc.commands.resource.resource_cmd import resource_delete
 from cgc.utils.requests_helper import call_api, EndpointTypes
 from cgc.utils.click_group import CustomCommand, CustomGroup
@@ -9,7 +12,7 @@ from cgc.utils.prepare_headers import get_api_url_and_prepare_headers
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
 from cgc.telemetry.basic import telemetry_permission_set
 from cgc.commands.compute.compute_responses import compute_logs_response
-from cgc.commands.auth.auth_cmd import auth_register
+# from cgc.commands.auth.auth_cmd import auth_register
 from cgc.utils import set_environment_data, check_if_config_exist, list_all_config_files
 from cgc.commands.cgc_helpers import table_of_user_context_files
 from cgc.utils.config_utils import config_path
@@ -37,7 +40,7 @@ def resource_events(app_name: str):
         request=EndpointTypes.get,
         url=url,
         headers=headers,
-        data=json.dumps(__payload).encode('utf-8'),
+        data=json.dumps(__payload).encode("utf-8"),
     )
     click.echo(
         compute_logs_response(retrieve_and_validate_response_send_metric(__res, metric))
@@ -122,3 +125,25 @@ def folder_of_contexts():
 def get_env_path():
     """Displays current environment file path"""
     click.echo(f"Current environment file path: {ENV_FILE_PATH}")
+
+
+@click.command("logs", cls=CustomCommand)
+@click.argument("app_name", type=click.STRING)
+def cgc_logs(app_name):
+    """Displays logs of a given app"""
+
+    if not app_name:
+        raise click.ClickException("Please provide a non-empty name")
+
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/resource/logs/{app_name}"
+    metric = "logs.get"
+    __res = call_api(
+        request=EndpointTypes.get,
+        url=url,
+        headers=headers,
+    )
+
+    click.echo(
+        cgc_logs_response(retrieve_and_validate_response_send_metric(__res, metric))
+    )

cgc/commands/cgc_cmd_responses.py

@@ -72,3 +72,11 @@ def cgc_status_response(data: dict):
         ),
         headers=list_headers,
     )
+
+
+def cgc_logs_response(data: dict):
+    return "\n".join(
+        "==> %s/%s <==\n%s" % (pod, cont, log)
+        for pod, containers in data["details"]["logs"].items()
+        for cont, log in containers.items()
+    )

cgc/commands/cgc_models.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class CGCEntityList(Enum):
+    """Base class for other lists"""
+
+    @classmethod
+    def get_list(cls) -> list[str]:
+        return [el.value for el in cls]

cgc/commands/compute/compute_models.py

@@ -1,12 +1,4 @@
-from enum import Enum
-
-
-class CGCEntityList(Enum):
-    """Base class for other lists"""
-
-    @classmethod
-    def get_list(cls) -> list[str]:
-        return [el.value for el in cls]
+from cgc.commands.cgc_models import CGCEntityList
 
 
 class ComputesList(CGCEntityList):
@@ -29,20 +21,6 @@ class ComputesList(CGCEntityList):
     CUSTOM = "custom"
 
 
-class DatabasesList(CGCEntityList):
-    """List of templates in cgc-server
-
-    :param Enum: name of template
-    :type Enum: str
-    """
-
-    MONGODB = "mongodb"
-    POSTGRESQL = "postgresql"
-    REDIS = "redis"
-    WEAVIATE = "weaviate"
-    # MINIO = "minio"
-
-
 class GPUsList(CGCEntityList):
     """List of templates in cgc-server
 
@@ -54,3 +32,5 @@ class GPUsList(CGCEntityList):
     A100 = "A100"
     A5000 = "A5000"
     H100 = "H100"
+    P40 = "P40"
+    P100 = "P100"

cgc/commands/compute/compute_utills.py

@@ -1,6 +1,6 @@
-from ast import main
 import cgc.utils.consts.env_consts as env_consts
 
+
 def list_get_mounted_volumes_paths(volume_list: list) -> str:
     """Formats and returns list of PVC volumes mounted to an app.
 
@@ -20,6 +20,7 @@ def list_get_mounted_volumes_paths(volume_list: list) -> str:
     )
     return volumes_mounted
 
+
 def list_get_mounted_volumes(volume_list: list) -> str:
     """Formats and returns list of PVC volumes mounted to an app.
 
@@ -39,16 +40,21 @@ def list_get_mounted_volumes(volume_list: list) -> str:
     )
     return volumes_mounted
 
-def get_app_mounts(pod_list:list) -> list:
+
+def get_app_mounts(pod_list: list) -> list:
     output_data = []
-    
+
     for pod in pod_list:
         try:
-            main_container_name = pod["labels"]["entity"]   
+            main_container_name = pod["labels"]["entity"]
             try:
-                main_container = [x for x in pod["containers"] if x["name"] == main_container_name][0]
+                main_container = [
+                    x for x in pod["containers"] if x["name"] == main_container_name
+                ][0]
             except IndexError:
-                raise Exception("Parser was unable to find main container in server output in container list")
+                raise Exception(
+                    "Parser was unable to find main container in server output in container list"
+                )
             volumes_mounted = list_get_mounted_volumes(main_container["mounts"])
             volumes_paths = list_get_mounted_volumes_paths(main_container["mounts"])
             pod_data = {
@@ -63,6 +69,7 @@ def get_app_mounts(pod_list:list) -> list:
             pass
     return output_data
 
+
 def get_app_list(pod_list: list, detailed: bool) -> list:
     """Formats and returns list of apps to print.
 
@@ -110,6 +117,9 @@ def get_app_list(pod_list: list, detailed: bool) -> list:
                 pod["labels"]["url"] = pod["labels"]["pod_url"]
                 pod["labels"].pop("app-token")
             pod["labels"].pop("pod_url")
+            pod["labels"].pop("resource-type")
+            pod["labels"].pop("api-key-id", None)
+            pod["labels"].pop("user-id", None)
 
             # appending the rest of labels
             pod_data.update(pod["labels"])

cgc/commands/db/db_cmd.py

@@ -2,9 +2,10 @@ import json
 import click
 
 from cgc.commands.compute.compute_responses import compute_list_response
-from cgc.commands.compute.compute_models import DatabasesList
+from cgc.commands.db.db_models import DatabasesList
 from cgc.commands.compute.compute_responses import compute_create_response
 from cgc.commands.compute.compute_utills import compute_create_payload
+from cgc.commands.exceptions import DatabaseCreationException
 from cgc.commands.resource.resource_cmd import resource_delete
 from cgc.utils.prepare_headers import get_api_url_and_prepare_headers
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
@@ -66,6 +67,11 @@ def db_create(
     :param name: name of app
     :type name: str
     """
+    try:
+        DatabasesList.verify(entity)
+    except DatabaseCreationException as e:
+        click.echo(e, err=True, color="yellow")
+        return
     api_url, headers = get_api_url_and_prepare_headers()
     url = f"{api_url}/v1/api/resource/create"
     metric = "db.create"

cgc/commands/db/db_models.py

@@ -0,0 +1,37 @@
+from cgc.commands.cgc_models import CGCEntityList
+from cgc.commands.exceptions import DatabaseCreationException
+
+
+class DatabasesList(CGCEntityList):
+    """List of templates in cgc-server
+
+    :param Enum: name of template
+    :type Enum: str
+    """
+
+    POSTGRESQL = "postgresql"
+    WEAVIATE = "weaviate"
+    # MINIO = "minio"
+    MONGODB = "mongodb"
+    REDIS = "redis"
+
+    @staticmethod
+    def verify(entity: str) -> str:
+        if entity == "mongodb":
+            raise DatabaseCreationException(
+                """
+Due to license agreement we can not serve MongoDB as a single click app.
+If you like to use it you can spawn one as custom image.
+
+cgc compute create custom -n name -c 4 -m 8 --image mongo
+"""
+            )
+        elif entity == "redis":
+            raise DatabaseCreationException(
+                """
+Due to license agreement we can not serve Redis as a single click app.
+If you like to use it you can spawn one as custom image.
+
+cgc compute create custom -n name -c 4 -m 8 --image redis
+"""
+            )

cgc/commands/exceptions.py

@@ -1,2 +1,10 @@
 class ResponseException(Exception):
     pass
+
+
+class InternalException(Exception):
+    pass
+
+
+class DatabaseCreationException(InternalException):
+    pass

cgc/commands/jobs/__init__.py

@@ -0,0 +1,10 @@
+from cgc.commands.exceptions import ResponseException
+
+
+class JobCommandException(ResponseException):
+    pass
+
+
+class NoJobsToList(JobCommandException):
+    def __init__(self) -> None:
+        super().__init__("No jobs to list.")

cgc/commands/jobs/job_utils.py

@@ -0,0 +1,180 @@
+from typing import Optional
+from cgc.commands.compute.compute_utills import list_get_mounted_volumes
+import cgc.utils.consts.env_consts as env_consts
+
+
+def job_delete_payload(name):
+    """
+    Create payload for job delete.
+    """
+    payload = {
+        "name": name,
+    }
+    return payload
+
+
+def job_create_payload(
+    name,
+    cpu,
+    memory,
+    volumes: list,
+    volume_full_path: str,
+    resource_data: list = [],
+    config_maps_data: list = [],
+    gpu: int = 0,
+    gpu_type: str = None,
+    shm_size: int = 0,
+    image_name: str = "",
+    startup_command: str = "",
+    repository_secret: str = "",
+    ttl_seconds_after_finished: Optional[int] = None,
+):
+    """
+    Create payload for app creation.
+    """
+    extra_payload = {}
+    if shm_size is not None and shm_size != 0:
+        extra_payload["shared_memory"] = shm_size
+
+    if ttl_seconds_after_finished is not None:
+        extra_payload["ttl_seconds_after_finished"] = ttl_seconds_after_finished
+
+    payload = {
+        "resource_data": {
+            "name": name,
+            "cpu": cpu,
+            "gpu": gpu,
+            "memory": memory,
+            "gpu_type": gpu_type,
+            "full_mount_path": volume_full_path,
+            **extra_payload,
+        }
+    }
+    try:
+        if len(volumes) != 0:
+            if not volume_full_path:
+                payload["resource_data"]["pv_volume"] = volumes
+            elif volume_full_path and len(volumes) != 1:
+                raise Exception(
+                    "Volume full path can only be used with a single volume"
+                )
+            else:
+                payload["resource_data"]["pv_volume"] = volumes
+    except TypeError:
+        pass
+    try:
+        resource_data_dict = {"resource_data": {}}
+        if len(resource_data) != 0:
+            for resource in resource_data:
+                try:
+                    key, value = resource.split("=")
+                    resource_data_dict["resource_data"][key] = value
+                except ValueError:
+                    raise Exception(
+                        "Invalid resource data format. Use key=value format"
+                    )
+        if image_name:
+            resource_data_dict["resource_data"]["custom_image"] = image_name
+        if startup_command:
+            resource_data_dict["resource_data"]["custom_command"] = startup_command
+        if repository_secret:
+            resource_data_dict["resource_data"][
+                "image_pull_secret_name"
+            ] = repository_secret
+        if resource_data_dict["resource_data"] != {}:
+            payload["template_specific_data"] = resource_data_dict
+    except TypeError:
+        pass
+    try:
+        if len(config_maps_data) != 0:
+            config_maps_data_dict = {}
+            for config_map in config_maps_data:
+                try:
+                    key, value = config_map.split(
+                        "="
+                    )  # where key is name of config map and value is data
+                    config_maps_data_dict[key] = (
+                        value  # value is dict, ex.: {"key": "value"}
+                    )
+                except ValueError:
+                    raise Exception(
+                        "Invalid config map data format. Use key=value format"
+                    )
+            payload["config_maps_data"] = config_maps_data_dict
+    except TypeError:
+        pass
+    return payload
+
+
+def get_job_list(job_list: list, job_pod_list: list):
+    list_of_json_job_data = get_job_json_data(job_list)
+
+    for i, job_data in enumerate(job_list):
+        list_of_json_job_data[i]["name"] = job_data.get("name", "")
+        list_of_json_job_data[i]["ttl_seconds_after_finished"] = job_data.get(
+            "ttl_seconds_after_finished", "N/A"
+        )
+    for job in list_of_json_job_data:
+        for job_pod in job_pod_list:
+            if job_pod.get("labels", {}).get("app-name") == job.get("name"):
+                job["status"] = job_pod.get("status", "Pending")
+                # job["status_reason"] = [
+                #     x.get("reason", "N/A") for x in job_pod.get("status_reasons", [])
+                # ]
+                # job["status_message"] = [
+                #     x.get("message", "N/A") for x in job_pod.get("status_reasons", [])
+                # ]
+                break
+
+    return list_of_json_job_data
+
+
+def get_job_json_data(job_list: list):
+    """Formats and returns list of jobs to print.
+
+    :param job_list: list of jobs
+    :type job_list: list
+    :return: formatted list of jobs
+    :rtype: list
+    """
+    output_data = []
+
+    for job in job_list:
+        try:
+            main_container_name = "custom-job"
+            try:
+                main_container = [
+                    x
+                    for x in job.get("containers", [])
+                    if x.get("name") == main_container_name
+                ][0]
+            except IndexError:
+                raise Exception(
+                    "Parser was unable to find main container in server output in container list"
+                )
+            volumes_mounted = list_get_mounted_volumes(main_container.get("mounts", []))
+            limits = main_container.get("resources", {}).get("limits")
+            cpu = limits.get("cpu") if limits is not None else 0
+            ram = limits.get("memory") if limits is not None else "0Gi"
+
+            job_data = {
+                "name": job.get("labels", {}).get("app-name"),
+                "status": job.get("status", {}).get("phase", "Pending"),
+                "volumes_mounted": volumes_mounted,
+                "cpu": cpu,
+                "ram": ram,
+                "gpu-count": job.get("labels", {}).get("gpu-count", 0),
+                "gpu-label": job.get("labels", {}).get("gpu-label", "N/A"),
+            }
+            # getting rid of unwanted and used values
+            if "pod-template-hash" in job["labels"].keys():
+                job["labels"].pop("pod-template-hash")
+            job["labels"].pop("entity")
+
+            # appending the rest of labels
+            job_data.update(job["labels"])
+            output_data.append(job_data)
+        except KeyError:
+            pass
+
+    return output_data