cgcsdk 1.0.7__py3-none-any.whl → 1.0.10__py3-none-any.whl

cgc/.env

@@ -6,4 +6,4 @@ CONFIG_FILE_NAME = cfg.json
 TMP_DIR = .tmp
 RELEASE = 1
 MAJOR_VERSION = 0
-MINOR_VERSION = 7
+MINOR_VERSION = 10

cgc/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Change Log
 
+## 1.0.10
+
+Release on April 24, 2024
+
+* added new command group: cgc secrets
+  * add - add new secret
+  * list - list all secrets in the namespace
+  * delete - delete secret (requires to be owner of the secret)
+  * update - update secret (requires to be owner of the secret)
+
+## 1.0.9
+
+Release on April 23, 2024
+
+* updated cgc to work with cgc-server 1.0.10
+* updated registration process
+* updated available apps, so it reflects changes in cgc-server
+* hotfix: listening jobs, which are not Running
+
+## 1.0.8
+
+Release on April 11, 2024
+
+* added new command group: cgc keys
+  * ssh create - create new ssh key, for ssh access
+  * ssh list - list all ssh keys
+  * ssh delete - delete ssh key
+  * ssh update - update ssh key
+
 ## 1.0.7
 
 Release on April 04, 2024

cgc/cgc.py

@@ -19,6 +19,8 @@ from cgc.commands.cgc_cmd import (
     resource_events,
     context_group,
 )
+from cgc.commands.user.keys_cmd import keys_group
+from cgc.commands.user.secret_cmd import secret_group
 
 from cgc.utils.version_control import check_version, _get_version
 from cgc.utils.click_group import CustomGroup
@@ -46,6 +48,8 @@ cli.add_command(cgc_status)
 cli.add_command(sending_telemetry_permission)
 cli.add_command(cgc_logs)
 cli.add_command(job_group)
+cli.add_command(keys_group)
+cli.add_command(secret_group)
 
 if __name__ == "__main__" or __name__ == "cgc.cgc":
     cli()

cgc/commands/auth/__init__.py

@@ -7,9 +7,9 @@ class AuthCommandException(ResponseException):
 
 class NoNamespaceInConfig(AuthCommandException):
     def __init__(self) -> None:
-        super().__init__(f"Namespace not readable from config file.")
+        super().__init__("Namespace not readable from config file.")
 
 
 class NoConfigFileFound(AuthCommandException):
     def __init__(self) -> None:
-        super().__init__(f"Config does not exists.")
+        super().__init__("Config does not exists.")

cgc/commands/auth/auth_cmd.py

@@ -9,7 +9,11 @@ from cgc.commands.auth.auth_utils import (
     auth_delete_api_key,
     auth_list_api_keys,
 )
-from cgc.utils.prepare_headers import get_url_and_prepare_headers_register
+from cgc.utils.prepare_headers import (
+    get_url_and_prepare_headers_register,
+    load_user_api_url,
+    load_user_cgc_secret,
+)
 from cgc.utils.cryptography import rsa_crypto
 from cgc.utils.click_group import CustomCommand, CustomGroup
 from cgc.utils.requests_helper import call_api, EndpointTypes
@@ -20,6 +24,7 @@ from cgc.utils.response_utils import (
 )
 from cgc.utils import (
     check_if_config_exist,
+    require_answer_loop,
     require_confirm_loop,
     find_first_available_config_name,
 )
@@ -55,9 +60,14 @@ def auth_register(config_filename: str = "cfg.json"):
         require_confirm_loop("Do you want to add new context?")
         config_filename = find_first_available_config_name()
 
+    cgc_api_url = require_answer_loop("Enter CGC server address", load_user_api_url())
+    cgc_secret = require_answer_loop("Enter CGC secret", load_user_cgc_secret())
+
     user_id = input("User ID: ")
     access_key = input("Access key: ")
-    url, headers = get_url_and_prepare_headers_register(user_id, access_key)
+    url, headers = get_url_and_prepare_headers_register(
+        user_id, access_key, cgc_api_url, cgc_secret
+    )
     metric = "auth.register"
     pub_key_bytes, priv_key_bytes = rsa_crypto.key_generate_pair()
     __payload = pub_key_bytes
@@ -74,6 +84,8 @@ def auth_register(config_filename: str = "cfg.json"):
             user_id,
             priv_key_bytes,
             config_filename,
+            cgc_api_url,
+            cgc_secret,
         )
     )
 

cgc/commands/auth/auth_responses.py

@@ -8,14 +8,21 @@ from cgc.utils.message_utils import key_error_decorator_for_helpers
 
 
 @key_error_decorator_for_helpers
-def auth_register_response(response, user_id, priv_key_bytes, config_filename) -> str:
+def auth_register_response(
+    response, user_id, priv_key_bytes, config_filename, cgc_api_url, cgc_secret
+) -> str:
     TMP_DIR_PATH = os.path.join(get_config_path(), TMP_DIR)
     unzip_dir, namespace = auth_utils.save_and_unzip_file(response)
     aes_key, password = auth_utils.get_aes_key_and_password(unzip_dir, priv_key_bytes)
 
     os.environ["CONFIG_FILE_NAME"] = config_filename
     save_to_config(
-        user_id=user_id, password=password, aes_key=aes_key, namespace=namespace
+        user_id=user_id,
+        password=password,
+        aes_key=aes_key,
+        namespace=namespace,
+        cgc_api_url=cgc_api_url,
+        cgc_secret=cgc_secret,
     )
     auth_utils.auth_create_api_key_with_save()
     shutil.rmtree(TMP_DIR_PATH)

cgc/commands/auth/auth_utils.py

@@ -10,7 +10,7 @@ from cgc.utils.config_utils import save_to_config
 from cgc.utils.config_utils import read_from_cfg
 from cgc.utils.cryptography import rsa_crypto
 from cgc.utils import prepare_headers
-from cgc.utils.consts.env_consts import API_URL, TMP_DIR
+from cgc.utils.consts.env_consts import TMP_DIR
 
 from cgc.utils.requests_helper import call_api, EndpointTypes
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
@@ -29,9 +29,10 @@ def _get_jwt_from_server(user_id: str = None, password: str = None) -> str:
     if user_id is None or password is None:
         if user_id is not None or password is not None:
             raise ValueError("Both user_id and password must be provided")
+    if user_id is None and password is None:
         user_id = urllib.parse.quote(read_from_cfg("user_id"))
         password = urllib.parse.quote(read_from_cfg("password"))
-    url = f"{API_URL}/v1/api/user/create/token"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/create/token"
     headers = {
         "accept": "application/json",
         "Content-Type": "application/x-www-form-urlencoded",
@@ -54,7 +55,7 @@ def get_jwt(user_id: str = None, password: str = None) -> str:
 
 def auth_create_api_key_with_save(user_id: str = None, password: str = None):
     """Function to create API key and API secret for user and save it to config file."""
-    url = f"{API_URL}/v1/api/user/create/api-key"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/create/api-key"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(
@@ -74,7 +75,7 @@ def auth_create_api_key_with_save(user_id: str = None, password: str = None):
 
 def auth_delete_api_key(api_key: str, user_id: str = None, password: str = None):
     """Function to delete API key."""
-    url = f"{API_URL}/v1/api/user/delete/api-key?api_key={api_key}"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/delete/api-key?api_key={api_key}"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(
@@ -89,7 +90,7 @@ def auth_delete_api_key(api_key: str, user_id: str = None, password: str = None)
 
 def auth_list_api_keys(user_id: str = None, password: str = None):
     """Function to list API keys."""
-    url = f"{API_URL}/v1/api/user/list/api-key"
+    url = f"{prepare_headers.load_user_api_url()}/v1/api/user/list/api-key"
     headers = prepare_headers.prepare_headers_api_key(user_id, password)
     metric = "auth.api-key"
     __res = call_api(

cgc/commands/cgc_models.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class CGCEntityList(Enum):
+    """Base class for other lists"""
+
+    @classmethod
+    def get_list(cls) -> list[str]:
+        return [el.value for el in cls]

cgc/commands/compute/compute_cmd.py

@@ -10,7 +10,7 @@ from cgc.commands.compute.compute_responses import (
     compute_list_response,
     get_compute_port_list,
 )
-from cgc.commands.compute.compute_utills import (
+from cgc.commands.compute.compute_utils import (
     compute_create_payload,
     port_delete_payload,
     port_modification_payload,

cgc/commands/compute/compute_models.py

@@ -1,12 +1,4 @@
-from enum import Enum
-
-
-class CGCEntityList(Enum):
-    """Base class for other lists"""
-
-    @classmethod
-    def get_list(cls) -> list[str]:
-        return [el.value for el in cls]
+from cgc.commands.cgc_models import CGCEntityList
 
 
 class ComputesList(CGCEntityList):
@@ -29,20 +21,6 @@ class ComputesList(CGCEntityList):
     CUSTOM = "custom"
 
 
-class DatabasesList(CGCEntityList):
-    """List of templates in cgc-server
-
-    :param Enum: name of template
-    :type Enum: str
-    """
-
-    MONGODB = "mongodb"
-    POSTGRESQL = "postgresql"
-    REDIS = "redis"
-    WEAVIATE = "weaviate"
-    # MINIO = "minio"
-
-
 class GPUsList(CGCEntityList):
     """List of templates in cgc-server
 

cgc/commands/compute/compute_responses.py

@@ -1,6 +1,6 @@
 from cgc.telemetry.basic import change_gauge
 from cgc.telemetry.basic import setup_gauge
-from cgc.commands.compute.compute_utills import get_app_list, get_app_mounts
+from cgc.commands.compute.compute_utils import get_app_list, get_app_mounts
 from cgc.utils.config_utils import get_namespace
 from cgc.utils.message_utils import key_error_decorator_for_helpers
 from cgc.commands.compute import NoAppsToList

cgc/commands/db/db_cmd.py

@@ -2,9 +2,10 @@ import json
 import click
 
 from cgc.commands.compute.compute_responses import compute_list_response
-from cgc.commands.compute.compute_models import DatabasesList
+from cgc.commands.db.db_models import DatabasesList
 from cgc.commands.compute.compute_responses import compute_create_response
-from cgc.commands.compute.compute_utills import compute_create_payload
+from cgc.commands.compute.compute_utils import compute_create_payload
+from cgc.commands.exceptions import DatabaseCreationException
 from cgc.commands.resource.resource_cmd import resource_delete
 from cgc.utils.prepare_headers import get_api_url_and_prepare_headers
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
@@ -66,6 +67,11 @@ def db_create(
     :param name: name of app
     :type name: str
     """
+    try:
+        DatabasesList.verify(entity)
+    except DatabaseCreationException as e:
+        click.echo(e, err=True, color="yellow")
+        return
     api_url, headers = get_api_url_and_prepare_headers()
     url = f"{api_url}/v1/api/resource/create"
     metric = "db.create"

cgc/commands/db/db_models.py

@@ -0,0 +1,37 @@
+from cgc.commands.cgc_models import CGCEntityList
+from cgc.commands.exceptions import DatabaseCreationException
+
+
+class DatabasesList(CGCEntityList):
+    """List of templates in cgc-server
+
+    :param Enum: name of template
+    :type Enum: str
+    """
+
+    POSTGRESQL = "postgresql"
+    WEAVIATE = "weaviate"
+    # MINIO = "minio"
+    MONGODB = "mongodb"
+    REDIS = "redis"
+
+    @staticmethod
+    def verify(entity: str) -> str:
+        if entity == "mongodb":
+            raise DatabaseCreationException(
+                """
+Due to license agreement we can not serve MongoDB as a single click app.
+If you like to use it you can spawn one as custom image.
+
+cgc compute create custom -n name -c 4 -m 8 --image mongo
+"""
+            )
+        elif entity == "redis":
+            raise DatabaseCreationException(
+                """
+Due to license agreement we can not serve Redis as a single click app.
+If you like to use it you can spawn one as custom image.
+
+cgc compute create custom -n name -c 4 -m 8 --image redis
+"""
+            )

cgc/commands/exceptions.py

@@ -1,2 +1,10 @@
 class ResponseException(Exception):
     pass
+
+
+class InternalException(Exception):
+    pass
+
+
+class DatabaseCreationException(InternalException):
+    pass

cgc/commands/jobs/job_utils.py

@@ -1,5 +1,5 @@
 from typing import Optional
-from cgc.commands.compute.compute_utills import list_get_mounted_volumes
+from cgc.commands.compute.compute_utils import list_get_mounted_volumes
 import cgc.utils.consts.env_consts as env_consts
 
 
@@ -106,35 +106,46 @@ def job_create_payload(
     return payload
 
 
-def get_job_list(job_pod_list: list, job_list: list):
-    list_of_json_data = get_job_pod_list(job_pod_list)
-    for json_data in list_of_json_data:
-        for job in job_list:
-            if job.get("name") == json_data.get("name"):
-                json_data["ttl_seconds_after_finished"] = job.get(
-                    "ttl_seconds_after_finished", "N/A"
-                )
+def get_job_list(job_list: list, job_pod_list: list):
+    list_of_json_job_data = get_job_json_data(job_list)
+
+    for i, job_data in enumerate(job_list):
+        list_of_json_job_data[i]["name"] = job_data.get("name", "")
+        list_of_json_job_data[i]["ttl_seconds_after_finished"] = job_data.get(
+            "ttl_seconds_after_finished", "N/A"
+        )
+    for job in list_of_json_job_data:
+        for job_pod in job_pod_list:
+            if job_pod.get("labels", {}).get("app-name") == job.get("name"):
+                job["status"] = job_pod.get("status", "Pending")
+                # job["status_reason"] = [
+                #     x.get("reason", "N/A") for x in job_pod.get("status_reasons", [])
+                # ]
+                # job["status_message"] = [
+                #     x.get("message", "N/A") for x in job_pod.get("status_reasons", [])
+                # ]
                 break
-    return list_of_json_data
+
+    return list_of_json_job_data
 
 
-def get_job_pod_list(job_pod_list: list) -> list:
+def get_job_json_data(job_list: list):
     """Formats and returns list of jobs to print.
 
-    :param pod_list: list of pods
-    :type pod_list: list
-    :return: formatted list of apps
+    :param job_list: list of jobs
+    :type job_list: list
+    :return: formatted list of jobs
     :rtype: list
     """
     output_data = []
 
-    for pod in job_pod_list:
+    for job in job_list:
         try:
             main_container_name = "custom-job"
             try:
                 main_container = [
                     x
-                    for x in pod.get("containers", [])
+                    for x in job.get("containers", [])
                     if x.get("name") == main_container_name
                 ][0]
             except IndexError:
@@ -146,29 +157,23 @@ def get_job_pod_list(job_pod_list: list) -> list:
             cpu = limits.get("cpu") if limits is not None else 0
             ram = limits.get("memory") if limits is not None else "0Gi"
 
-            pod_data = {
-                "name": pod.get("labels", {}).get("app-name"),
-                "status": pod.get("status", {}),
+            job_data = {
+                "name": job.get("labels", {}).get("app-name"),
+                "status": job.get("status", {}).get("phase", "Pending"),
                 "volumes_mounted": volumes_mounted,
                 "cpu": cpu,
                 "ram": ram,
-                "gpu-count": pod.get("labels", {}).get("gpu-count", 0),
-                "gpu-label": pod.get("labels", {}).get("gpu-label", "N/A"),
+                "gpu-count": job.get("labels", {}).get("gpu-count", 0),
+                "gpu-label": job.get("labels", {}).get("gpu-label", "N/A"),
             }
             # getting rid of unwanted and used values
-            if "pod-template-hash" in pod["labels"].keys():
-                pod["labels"].pop("pod-template-hash")
-            pod["labels"].pop("app-name")
-            pod["labels"].pop("entity")
-            pod["labels"].pop("resource-type")
-            pod["labels"].pop("job-name")
-            pod["labels"].pop("controller-uid")
-            pod["labels"].pop("api-key-id", None)
-            pod["labels"].pop("user-id", None)
+            if "pod-template-hash" in job["labels"].keys():
+                job["labels"].pop("pod-template-hash")
+            job["labels"].pop("entity")
 
             # appending the rest of labels
-            pod_data.update(pod["labels"])
-            output_data.append(pod_data)
+            job_data.update(job["labels"])
+            output_data.append(job_data)
         except KeyError:
             pass
 

cgc/commands/jobs/jobs_responses.py

@@ -32,7 +32,7 @@ def job_list_response(data: dict) -> list:
     if not job_list:
         raise NoJobsToList()
 
-    list_of_json_data = get_job_list(job_pod_list, job_list)
+    list_of_json_data = get_job_list(job_list, job_pod_list)
     table = fill_missing_values_in_a_response(list_of_json_data)
 
     return tabulate_a_response(table)

cgc/commands/resource/resource_cmd.py

@@ -1,7 +1,8 @@
 import click
 import json
 
-from cgc.commands.compute.compute_models import ComputesList, DatabasesList
+from cgc.commands.db.db_models import DatabasesList
+from cgc.commands.compute.compute_models import ComputesList
 from cgc.commands.compute.compute_responses import (
     template_list_response,
     template_get_start_path_response,
@@ -9,7 +10,7 @@ from cgc.commands.compute.compute_responses import (
     compute_delete_response,
 )
 
-from cgc.commands.compute.compute_utills import compute_delete_payload
+from cgc.commands.compute.compute_utils import compute_delete_payload
 from cgc.commands.resource.resource_responses import get_ingress_list_from_response
 
 from cgc.utils.prepare_headers import get_api_url_and_prepare_headers

cgc/commands/user/__init__.py

@@ -0,0 +1,14 @@
+from cgc.commands.exceptions import ResponseException
+
+
+class KeysCommandException(ResponseException):
+    """Base exception for all key commands."""
+
+
+class SecretsCommandException(ResponseException):
+    """Base exception for all secret commands."""
+
+
+class NoSecretsToList(SecretsCommandException):
+    def __init__(self) -> None:
+        super().__init__("No secrets to list.")

cgc/commands/user/keys_cmd.py

@@ -0,0 +1,179 @@
+import json
+import click
+from typing import Optional
+from cgc.commands.user.keys_responses import (
+    create_ssh_key_response,
+    delete_ssh_key_response,
+    list_ssh_keys_response,
+    update_ssh_key_response,
+)
+from cgc.commands.user.keys_utils import create_ssh_key_payload, update_ssh_key_payload
+from cgc.commands.user.keys_models import SSHKeyTypes
+from cgc.utils.prepare_headers import get_api_url_and_prepare_headers
+from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
+from cgc.utils.click_group import CustomGroup, CustomCommand
+from cgc.utils.requests_helper import call_api, EndpointTypes
+
+
+@click.group(name="keys", cls=CustomGroup)
+def keys_group():
+    """
+    Management of keys.
+    """
+
+
+@keys_group.group(name="ssh", cls=CustomGroup)
+def ssh_keys_group():
+    """
+    Management of ssh keys.
+    """
+
+
+@ssh_keys_group.command("create", cls=CustomCommand)
+@click.option(
+    "-pk",
+    "--pub-key",
+    "public_key",
+    type=click.STRING,
+    required=False,
+    help="Whole Public key string",
+)
+@click.option(
+    "-kt",
+    "--key-type",
+    "key_type",
+    type=click.Choice(SSHKeyTypes.get_list(), case_sensitive=False),
+    required=False,
+    # default="ssh-rsa",
+    help="Type of the key",
+)
+@click.option(
+    "-k",
+    "--key",
+    "key",
+    type=click.STRING,
+    required=False,
+    help="Public key",
+)
+@click.option(
+    "-c",
+    "--comment",
+    "comment",
+    type=click.STRING,
+    required=False,
+    default="",
+    help="Comment for the key",
+)
+def create_ssh_key(
+    public_key: Optional[str] = None,
+    key_type: Optional[str] = None,
+    key: Optional[str] = None,
+    comment: str = "",
+):
+    """Create a new SSH key"""
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/keys/ssh/create"
+    metric = "keys.create"
+    __payload = create_ssh_key_payload(public_key, key_type, key, comment)
+    __res = call_api(
+        request=EndpointTypes.post,
+        url=url,
+        headers=headers,
+        data=json.dumps(__payload).encode("utf-8"),
+    )
+    click.echo(
+        create_ssh_key_response(
+            retrieve_and_validate_response_send_metric(__res, metric)
+        ),
+        color="green",
+    )
+
+
+@ssh_keys_group.command("update", cls=CustomCommand)
+@click.argument("key_id", type=click.STRING)
+@click.option(
+    "-kt",
+    "--key-type",
+    "key_type",
+    type=click.Choice(SSHKeyTypes.get_list(), case_sensitive=False),
+    # default="ssh-rsa",
+    required=False,
+    help="Type of the key",
+)
+@click.option(
+    "-k",
+    "--key",
+    "key",
+    type=click.STRING,
+    required=False,
+    help="Public key",
+)
+@click.option(
+    "-c",
+    "--comment",
+    "comment",
+    type=click.STRING,
+    required=False,
+    help="Comment for the key",
+)
+def update_ssh_key(
+    key_id: str,
+    key_type: Optional[str] = None,
+    key: Optional[str] = None,
+    comment: Optional[str] = None,
+):
+    """Update an existing SSH key"""
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/keys/ssh/update/{key_id}"
+    metric = "keys.update"
+    __payload = update_ssh_key_payload(key_type, key, comment)
+    __res = call_api(
+        request=EndpointTypes.post,
+        url=url,
+        headers=headers,
+        data=json.dumps(__payload).encode("utf-8"),
+    )
+    click.echo(
+        update_ssh_key_response(
+            retrieve_and_validate_response_send_metric(__res, metric)
+        ),
+        color="green",
+    )
+
+
+@ssh_keys_group.command("delete", cls=CustomCommand)
+@click.argument("key_id", type=click.STRING)
+def delete_ssh_key(key_id: str):
+    """Delete an SSH key"""
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/keys/ssh/delete/{key_id}"
+    metric = "keys.delete"
+    __res = call_api(
+        request=EndpointTypes.delete,
+        url=url,
+        headers=headers,
+    )
+    click.echo(
+        delete_ssh_key_response(
+            retrieve_and_validate_response_send_metric(__res, metric)
+        ),
+        color="green",
+    )
+
+
+@ssh_keys_group.command("list", cls=CustomCommand)
+def list_ssh_keys():
+    """List all SSH keys"""
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/keys/ssh/list"
+    metric = "keys.list"
+    __res = call_api(
+        request=EndpointTypes.get,
+        url=url,
+        headers=headers,
+    )
+    click.echo(
+        list_ssh_keys_response(
+            retrieve_and_validate_response_send_metric(__res, metric)
+        )
+    )