cgcsdk 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

cgc/.env

@@ -6,4 +6,4 @@ CONFIG_FILE_NAME = cfg.json
 TMP_DIR = .tmp
 RELEASE = 1
 MAJOR_VERSION = 0
-MINOR_VERSION = 5
+MINOR_VERSION = 7

cgc/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change Log
 
+## 1.0.7
+
+Release on April 04, 2024
+
+* speed of resource creation with volumes / shared memory, has been improved
+* added jobs creation: cgc job create
+* added jobs list: cgc job list
+* added jobs delete: cgc job delete
+* added logs (STDOUT, STDERR) reader for spawned applications: cgc logs
+* added SDK for logs & jobs
+
+## 1.0.6
+
+Release on March 13, 2024
+
+* updated exception handling for SDK
+* fix for cgc CLI command: compute create
+* ensure encode("utf-8") for all payloads generated by CLI/SDK
+
 ## 1.0.5
 
 Release on March 7, 2024
@@ -11,7 +30,7 @@ Release on March 7, 2024
   * allows to mount volume with full path; works with one volume only
 * added support for adding for compute create config_map_data
 * added for compute create
-  * --startup-command flag to set startup command (overrides default command in Dockerfile)
+  * startup_command as a STDIN on second argument
   * --repository-secret flag to set docker registry secret from namespace
   * --image flag to set docker image for custom template
 * managing compute ports, require user confirmation
@@ -21,8 +40,7 @@ Release on March 7, 2024
 * add compute and db **list** to cgc sdk
 * add resource ready command to cgc sdk
 * on-premises support for node_port_enabled
-* added new command: cgc context replace-env "path_to_env_file"
-* added new command: cgc context get-env "where_to_save_env_file"
+* added new command: cgc context get-env-path
 * added new command: cgc compute list-mounts
 
 ## 1.0.4

cgc/cgc.py

@@ -4,6 +4,7 @@ from cgc.commands.compute.compute_cmd import compute_group
 from cgc.commands.billing.billing_cmd import billing_group
 from cgc.commands.db.db_cmd import db_group
 from cgc.commands.resource.resource_cmd import resource_group
+from cgc.commands.jobs.jobs_cmd import job_group
 from cgc.commands.auth.auth_cmd import (
     api_keys_group,
     auth_register,
@@ -13,10 +14,12 @@ from cgc.commands.debug.debug_cmd import debug_group
 from cgc.commands.cgc_cmd import (
     cgc_rm,
     cgc_status,
+    cgc_logs,
     sending_telemetry_permission,
     resource_events,
     context_group,
 )
+
 from cgc.utils.version_control import check_version, _get_version
 from cgc.utils.click_group import CustomGroup
 
@@ -41,7 +44,8 @@ cli.add_command(resource_group)
 cli.add_command(billing_group)
 cli.add_command(cgc_status)
 cli.add_command(sending_telemetry_permission)
-
+cli.add_command(cgc_logs)
+cli.add_command(job_group)
 
 if __name__ == "__main__" or __name__ == "cgc.cgc":
     cli()

cgc/commands/cgc_cmd.py

@@ -1,7 +1,10 @@
 import click
 import json
 
-from cgc.commands.cgc_cmd_responses import cgc_status_response
+from cgc.commands.cgc_cmd_responses import (
+    cgc_logs_response,
+    cgc_status_response,
+)
 from cgc.commands.resource.resource_cmd import resource_delete
 from cgc.utils.requests_helper import call_api, EndpointTypes
 from cgc.utils.click_group import CustomCommand, CustomGroup
@@ -9,7 +12,7 @@ from cgc.utils.prepare_headers import get_api_url_and_prepare_headers
 from cgc.utils.response_utils import retrieve_and_validate_response_send_metric
 from cgc.telemetry.basic import telemetry_permission_set
 from cgc.commands.compute.compute_responses import compute_logs_response
-from cgc.commands.auth.auth_cmd import auth_register
+# from cgc.commands.auth.auth_cmd import auth_register
 from cgc.utils import set_environment_data, check_if_config_exist, list_all_config_files
 from cgc.commands.cgc_helpers import table_of_user_context_files
 from cgc.utils.config_utils import config_path
@@ -37,7 +40,7 @@ def resource_events(app_name: str):
         request=EndpointTypes.get,
         url=url,
         headers=headers,
-        data=json.dumps(__payload),
+        data=json.dumps(__payload).encode("utf-8"),
     )
     click.echo(
         compute_logs_response(retrieve_and_validate_response_send_metric(__res, metric))
@@ -122,3 +125,25 @@ def folder_of_contexts():
 def get_env_path():
     """Displays current environment file path"""
     click.echo(f"Current environment file path: {ENV_FILE_PATH}")
+
+
+@click.command("logs", cls=CustomCommand)
+@click.argument("app_name", type=click.STRING)
+def cgc_logs(app_name):
+    """Displays logs of a given app"""
+
+    if not app_name:
+        raise click.ClickException("Please provide a non-empty name")
+
+    api_url, headers = get_api_url_and_prepare_headers()
+    url = f"{api_url}/v1/api/resource/logs/{app_name}"
+    metric = "logs.get"
+    __res = call_api(
+        request=EndpointTypes.get,
+        url=url,
+        headers=headers,
+    )
+
+    click.echo(
+        cgc_logs_response(retrieve_and_validate_response_send_metric(__res, metric))
+    )

cgc/commands/cgc_cmd_responses.py

@@ -72,3 +72,11 @@ def cgc_status_response(data: dict):
         ),
         headers=list_headers,
     )
+
+
+def cgc_logs_response(data: dict):
+    return "\n".join(
+        "==> %s/%s <==\n%s" % (pod, cont, log)
+        for pod, containers in data["details"]["logs"].items()
+        for cont, log in containers.items()
+    )

cgc/commands/cgc_helpers.py

@@ -31,15 +31,3 @@ def table_of_user_context_files(config_files: List[str]):
         contexts_sorted.append(contexts[contexts_nrs.index(context)])
 
     return tabulate(contexts_sorted, headers=headers)
-
-def process_stdin(input_file: str):
-    full_stdin = []
-    try:
-        for line in input_file:
-            line = line.strip()
-            if line and line != '|':
-                full_stdin.append(line.rstrip('\n'))  # Remove newline characters
-    except TypeError:
-        return None
-    return " ".join(full_stdin)
-            

cgc/commands/compute/compute_cmd.py

@@ -1,7 +1,6 @@
 import json
 import sys
 import click
-from cgc.commands.cgc_helpers import process_stdin
 
 from cgc.commands.compute.compute_models import ComputesList, GPUsList
 from cgc.commands.compute.compute_responses import (
@@ -61,7 +60,7 @@ def compute_filebrowser_create(puid: int, pgid: int):
         request=EndpointTypes.post,
         url=url,
         headers=headers,
-        data=json.dumps(__payload),
+        data=json.dumps(__payload).encode("utf-8"),
     )
     click.echo(
         compute_create_filebrowser_response(
@@ -119,7 +118,10 @@ def compute_port_add(app_name: str, port_name: str, port: int, ingress: bool):
         app_name=app_name,
     )
     __res = call_api(
-        request=EndpointTypes.post, url=url, headers=headers, data=json.dumps(__payload)
+        request=EndpointTypes.post,
+        url=url,
+        headers=headers,
+        data=json.dumps(__payload).encode("utf-8"),
     )
 
     __res_list = get_compute_port_list(
@@ -181,7 +183,10 @@ def compute_port_update(app_name: str, port_name: str, port: int, ingress: bool)
         app_name=app_name,
     )
     __res = call_api(
-        request=EndpointTypes.post, url=url, headers=headers, data=json.dumps(__payload)
+        request=EndpointTypes.post,
+        url=url,
+        headers=headers,
+        data=json.dumps(__payload).encode("utf-8"),
     )
 
     retrieve_and_validate_response_send_metric(__res, metric)
@@ -219,7 +224,10 @@ def compute_port_delete(app_name: str, port_name: str):
         app_name=app_name,
     )
     __res = call_api(
-        request=EndpointTypes.post, url=url, headers=headers, data=json.dumps(__payload)
+        request=EndpointTypes.post,
+        url=url,
+        headers=headers,
+        data=json.dumps(__payload).encode("utf-8"),
     )
 
     retrieve_and_validate_response_send_metric(__res, metric)
@@ -251,7 +259,7 @@ def compute_port_list(app_name: str):
 
 @compute_group.command("create", cls=CustomCommand)
 @click.argument("entity", type=click.Choice(ComputesList.get_list()))
-@click.argument("startup_command", type=click.File("r"), default="-", required=False)
+@click.argument("startup_command", required=False)
 @click.option(
     "-n", "--name", "name", type=click.STRING, required=True, help="Desired app name"
 )
@@ -393,8 +401,13 @@ def compute_create(
     """
     api_url, headers = get_api_url_and_prepare_headers()
     url = f"{api_url}/v1/api/resource/create"
-    if startup_command != "-":
-        startup_command = process_stdin(startup_command)
+    cleaned_data = ""
+    if not sys.stdin.isatty():
+        input_data = sys.stdin.read()
+        cleaned_data = input_data.replace("|", "")
+        startup_command = cleaned_data
+    elif startup_command:
+        cleaned_data = startup_command
     metric = "compute.create"
     __payload = compute_create_payload(
         name=name,
@@ -409,7 +422,7 @@ def compute_create(
         gpu_type=gpu_type,
         shm_size=shm_size,
         image_name=image_name,
-        startup_command=startup_command,
+        startup_command=cleaned_data,
         repository_secret=repository_secret,
         node_port_enabled=node_port_enabled,
     )
@@ -443,7 +456,7 @@ def compute_create(
         request=EndpointTypes.post,
         url=url,
         headers=headers,
-        data=json.dumps(__payload),
+        data=json.dumps(__payload).encode("utf-8"),
     )
     click.echo(
         compute_create_response(

cgc/commands/compute/compute_models.py

@@ -54,3 +54,5 @@ class GPUsList(CGCEntityList):
     A100 = "A100"
     A5000 = "A5000"
     H100 = "H100"
+    P40 = "P40"
+    P100 = "P100"

cgc/commands/compute/compute_utills.py

@@ -1,6 +1,6 @@
-from ast import main
 import cgc.utils.consts.env_consts as env_consts
 
+
 def list_get_mounted_volumes_paths(volume_list: list) -> str:
     """Formats and returns list of PVC volumes mounted to an app.
 
@@ -20,6 +20,7 @@ def list_get_mounted_volumes_paths(volume_list: list) -> str:
     )
     return volumes_mounted
 
+
 def list_get_mounted_volumes(volume_list: list) -> str:
     """Formats and returns list of PVC volumes mounted to an app.
 
@@ -39,16 +40,21 @@ def list_get_mounted_volumes(volume_list: list) -> str:
     )
     return volumes_mounted
 
-def get_app_mounts(pod_list:list) -> list:
+
+def get_app_mounts(pod_list: list) -> list:
     output_data = []
-    
+
     for pod in pod_list:
         try:
-            main_container_name = pod["labels"]["entity"]   
+            main_container_name = pod["labels"]["entity"]
             try:
-                main_container = [x for x in pod["containers"] if x["name"] == main_container_name][0]
+                main_container = [
+                    x for x in pod["containers"] if x["name"] == main_container_name
+                ][0]
             except IndexError:
-                raise Exception("Parser was unable to find main container in server output in container list")
+                raise Exception(
+                    "Parser was unable to find main container in server output in container list"
+                )
             volumes_mounted = list_get_mounted_volumes(main_container["mounts"])
             volumes_paths = list_get_mounted_volumes_paths(main_container["mounts"])
             pod_data = {
@@ -63,6 +69,7 @@ def get_app_mounts(pod_list:list) -> list:
             pass
     return output_data
 
+
 def get_app_list(pod_list: list, detailed: bool) -> list:
     """Formats and returns list of apps to print.
 
@@ -110,6 +117,9 @@ def get_app_list(pod_list: list, detailed: bool) -> list:
                 pod["labels"]["url"] = pod["labels"]["pod_url"]
                 pod["labels"].pop("app-token")
             pod["labels"].pop("pod_url")
+            pod["labels"].pop("resource-type")
+            pod["labels"].pop("api-key-id", None)
+            pod["labels"].pop("user-id", None)
 
             # appending the rest of labels
             pod_data.update(pod["labels"])

cgc/commands/db/db_cmd.py

@@ -80,7 +80,7 @@ def db_create(
         request=EndpointTypes.post,
         url=url,
         headers=headers,
-        data=json.dumps(__payload),
+        data=json.dumps(__payload).encode("utf-8"),
     )
     click.echo(
         compute_create_response(

cgc/commands/jobs/__init__.py

@@ -0,0 +1,10 @@
+from cgc.commands.exceptions import ResponseException
+
+
+class JobCommandException(ResponseException):
+    pass
+
+
+class NoJobsToList(JobCommandException):
+    def __init__(self) -> None:
+        super().__init__("No jobs to list.")

cgc/commands/jobs/job_utils.py

@@ -0,0 +1,175 @@
+from typing import Optional
+from cgc.commands.compute.compute_utills import list_get_mounted_volumes
+import cgc.utils.consts.env_consts as env_consts
+
+
+def job_delete_payload(name):
+    """
+    Create payload for job delete.
+    """
+    payload = {
+        "name": name,
+    }
+    return payload
+
+
+def job_create_payload(
+    name,
+    cpu,
+    memory,
+    volumes: list,
+    volume_full_path: str,
+    resource_data: list = [],
+    config_maps_data: list = [],
+    gpu: int = 0,
+    gpu_type: str = None,
+    shm_size: int = 0,
+    image_name: str = "",
+    startup_command: str = "",
+    repository_secret: str = "",
+    ttl_seconds_after_finished: Optional[int] = None,
+):
+    """
+    Create payload for app creation.
+    """
+    extra_payload = {}
+    if shm_size is not None and shm_size != 0:
+        extra_payload["shared_memory"] = shm_size
+
+    if ttl_seconds_after_finished is not None:
+        extra_payload["ttl_seconds_after_finished"] = ttl_seconds_after_finished
+
+    payload = {
+        "resource_data": {
+            "name": name,
+            "cpu": cpu,
+            "gpu": gpu,
+            "memory": memory,
+            "gpu_type": gpu_type,
+            "full_mount_path": volume_full_path,
+            **extra_payload,
+        }
+    }
+    try:
+        if len(volumes) != 0:
+            if not volume_full_path:
+                payload["resource_data"]["pv_volume"] = volumes
+            elif volume_full_path and len(volumes) != 1:
+                raise Exception(
+                    "Volume full path can only be used with a single volume"
+                )
+            else:
+                payload["resource_data"]["pv_volume"] = volumes
+    except TypeError:
+        pass
+    try:
+        resource_data_dict = {"resource_data": {}}
+        if len(resource_data) != 0:
+            for resource in resource_data:
+                try:
+                    key, value = resource.split("=")
+                    resource_data_dict["resource_data"][key] = value
+                except ValueError:
+                    raise Exception(
+                        "Invalid resource data format. Use key=value format"
+                    )
+        if image_name:
+            resource_data_dict["resource_data"]["custom_image"] = image_name
+        if startup_command:
+            resource_data_dict["resource_data"]["custom_command"] = startup_command
+        if repository_secret:
+            resource_data_dict["resource_data"][
+                "image_pull_secret_name"
+            ] = repository_secret
+        if resource_data_dict["resource_data"] != {}:
+            payload["template_specific_data"] = resource_data_dict
+    except TypeError:
+        pass
+    try:
+        if len(config_maps_data) != 0:
+            config_maps_data_dict = {}
+            for config_map in config_maps_data:
+                try:
+                    key, value = config_map.split(
+                        "="
+                    )  # where key is name of config map and value is data
+                    config_maps_data_dict[key] = (
+                        value  # value is dict, ex.: {"key": "value"}
+                    )
+                except ValueError:
+                    raise Exception(
+                        "Invalid config map data format. Use key=value format"
+                    )
+            payload["config_maps_data"] = config_maps_data_dict
+    except TypeError:
+        pass
+    return payload
+
+
+def get_job_list(job_pod_list: list, job_list: list):
+    list_of_json_data = get_job_pod_list(job_pod_list)
+    for json_data in list_of_json_data:
+        for job in job_list:
+            if job.get("name") == json_data.get("name"):
+                json_data["ttl_seconds_after_finished"] = job.get(
+                    "ttl_seconds_after_finished", "N/A"
+                )
+                break
+    return list_of_json_data
+
+
+def get_job_pod_list(job_pod_list: list) -> list:
+    """Formats and returns list of jobs to print.
+
+    :param pod_list: list of pods
+    :type pod_list: list
+    :return: formatted list of apps
+    :rtype: list
+    """
+    output_data = []
+
+    for pod in job_pod_list:
+        try:
+            main_container_name = "custom-job"
+            try:
+                main_container = [
+                    x
+                    for x in pod.get("containers", [])
+                    if x.get("name") == main_container_name
+                ][0]
+            except IndexError:
+                raise Exception(
+                    "Parser was unable to find main container in server output in container list"
+                )
+            volumes_mounted = list_get_mounted_volumes(main_container.get("mounts", []))
+            limits = main_container.get("resources", {}).get("limits")
+            cpu = limits.get("cpu") if limits is not None else 0
+            ram = limits.get("memory") if limits is not None else "0Gi"
+
+            pod_data = {
+                "name": pod.get("labels", {}).get("app-name"),
+                "status": pod.get("status", {}),
+                "volumes_mounted": volumes_mounted,
+                "cpu": cpu,
+                "ram": ram,
+                "gpu-count": pod.get("labels", {}).get("gpu-count", 0),
+                "gpu-label": pod.get("labels", {}).get("gpu-label", "N/A"),
+            }
+            # getting rid of unwanted and used values
+            if "pod-template-hash" in pod["labels"].keys():
+                pod["labels"].pop("pod-template-hash")
+            pod["labels"].pop("app-name")
+            pod["labels"].pop("entity")
+            pod["labels"].pop("resource-type")
+            pod["labels"].pop("job-name")
+            pod["labels"].pop("controller-uid")
+            pod["labels"].pop("api-key-id", None)
+            pod["labels"].pop("user-id", None)
+
+            # appending the rest of labels
+            pod_data.update(pod["labels"])
+            output_data.append(pod_data)
+        except KeyError:
+            pass
+
+    return output_data