kubernetes-watch 0.1.0__py3-none-any.whl

kube_watch/enums/kube.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class Hosts(str, Enum):
+    LOCAL = 'local'
+    REMOTE = 'remote'

kube_watch/enums/logic.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+class Operations(str, Enum):
+    OR  = 'or'
+    AND = 'and'
+    SUM = 'sum'
+    AVG = 'avg'
+    MAX = 'max'
+    MIN = 'min'

kube_watch/enums/providers.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+class Providers(str, Enum):
+    AWS   = "aws"
+    AZURE = "azure"
+    GCP   = "gcp"
+    VAULT = "vault"
+
+
+class AwsResources(str, Enum):
+    ECR = "ecr" # elastic container registry
+    S3  = "s3"
+    IAM = "iam" 

kube_watch/enums/workflow.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+class ParameterType(str, Enum):
+    STATIC = 'static'
+    FROM_ENV = 'env'
+
+
+class TaskRunners(str, Enum):
+    SEQUENTIAL = 'sequential'
+    CONCURRENT = 'concurrent'
+    DASK       = 'dask'
+    RAY        = 'ray'
+
+
+class TaskInputsType(str, Enum):
+    ARG = 'arg'
+    DICT = 'dict'

kube_watch/models/common.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, ConfigDict
+from humps.camel import case
+
+def to_camel(string):
+    if string == "id":
+        return "_id"
+    if string.startswith("_"):  # "_id"
+        return string
+    return case(string)
+
+class CamelModel(BaseModel):
+    """
+    Replacement for pydanitc BaseModel which simply adds a camel case alias to every field
+    NOTE: This has been updated for Pydantic 2 to remove some common encoding helpers
+    """
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)

kube_watch/models/workflow.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, Dict, Any
+from kube_watch.enums.workflow import ParameterType, TaskRunners, TaskInputsType
+from kube_watch.enums.logic import Operations
+
+from .common import CamelModel
+
+class Parameter(CamelModel):
+    name: str
+    value: Any
+    type: Optional[ParameterType] = ParameterType.STATIC
+
+class Artifact(CamelModel):
+    path: str
+
+class Inputs(CamelModel):
+    parameters: Optional[List[Parameter]] = []
+    artifacts: Optional[List[Artifact]] = []
+
+class Dependency(CamelModel):
+    taskName: str
+    inputParamName: Optional[str] = None
+
+class Condition(CamelModel):
+    tasks: List[str]
+    operation: Optional[Operations] = Operations.AND
+
+class Task(CamelModel):
+    module: str
+    task: str
+    name: str
+    inputsArgType: Optional[TaskInputsType] = TaskInputsType.ARG
+    inputs: Optional[Inputs] = None
+    dependency: Optional[List[Dependency]]  = None
+    conditional: Optional[Condition]  = None
+    outputs: Optional[List[str]]  = None
+
+class WorkflowConfig(CamelModel):
+    name: str
+    runner: TaskRunners = TaskRunners.CONCURRENT
+    tasks: List[Task]
+
+class WorkflowOutput(CamelModel):
+    flow_run: Any
+    config: Any
+
+class BatchFlowItem(CamelModel):
+    path: str
+
+class BatchFlowConfig(CamelModel):
+    # Only possible runners are concurrent and sequential
+    runner: TaskRunners = TaskRunners.CONCURRENT
+    items: List[BatchFlowItem]
+
+
+

kube_watch/modules/clusters/kube.py

@@ -0,0 +1,186 @@
+from prefect import get_run_logger
+from typing import List
+from kubernetes import config
+from kubernetes import client
+from kubernetes.client.rest import ApiException
+import base64
+import datetime
+
+from kube_watch.enums.kube import Hosts
+
+logger = get_run_logger()
+
+    
+def setup(host=Hosts.REMOTE, context=None):
+    if host == Hosts.LOCAL:
+        # Running outside a Kubernetes cluster (e.g., local development)
+        config.load_kube_config(context=context)  # You can specify the context here if necessary   
+    else:
+        # Running inside a Kubernetes cluster
+        config.load_incluster_config()
+
+
+
+def create_or_update_configmap(config_name, namespace, data):
+    """
+    Create or update a ConfigMap in a specified namespace if the data is different.
+    
+    :param config_name: The name of the ConfigMap.
+    :param namespace: The namespace of the ConfigMap.
+    :param data: A dictionary containing the data for the ConfigMap.
+    :return: True if the ConfigMap was created or updated, False otherwise.
+    """
+    v1 = client.CoreV1Api()
+    configmap_metadata = client.V1ObjectMeta(name=config_name, namespace=namespace)
+    configmap = client.V1ConfigMap(api_version="v1", kind="ConfigMap", metadata=configmap_metadata, data=data)
+    
+    try:
+        existing_configmap = v1.read_namespaced_config_map(name=config_name, namespace=namespace)
+        # Compare the existing ConfigMap's data with the new data
+        if existing_configmap.data == data:
+            logger.info("No update needed for ConfigMap: {}".format(config_name))
+            return False
+        else:
+            # Data is different, update the ConfigMap
+            api_response = v1.replace_namespaced_config_map(name=config_name, namespace=namespace, body=configmap)
+            logger.info("ConfigMap updated. Name: {}".format(api_response.metadata.name))
+            return True
+    except ApiException as e:
+        if e.status == 404:  # ConfigMap not found, create it
+            try:
+                api_response = v1.create_namespaced_config_map(namespace=namespace, body=configmap)
+                logger.info("ConfigMap created. Name: {}".format(api_response.metadata.name))
+                return {'trigger_restart': True}
+            except ApiException as e:
+                logger.error("Exception when creating ConfigMap: {}".format(e))
+                raise ValueError
+        else:
+            logger.error("Failed to get or create ConfigMap: {}".format(e))
+            raise ValueError
+        
+
+def create_or_update_secret(secret_name, namespace, data, secret_type = None):
+    """
+    Create or update a Secret in a specified namespace if the data is different.
+    
+    :param name: The name of the Secret.
+    :param namespace: The namespace of the Secret.
+    :param data: A dictionary containing the data for the Secret. Values must be strings (not Base64 encoded).
+    :return: True if the secret was created or updated, False otherwise.
+    """
+    if secret_type == None:
+        secret_type = "Opaque"
+
+    v1 = client.CoreV1Api()
+    secret_metadata = client.V1ObjectMeta(name=secret_name, namespace=namespace)
+    secret = client.V1Secret(
+        api_version="v1", 
+        kind="Secret", 
+        metadata=secret_metadata, 
+        string_data=data,
+        type=secret_type
+    )
+    
+    try:
+        existing_secret = v1.read_namespaced_secret(name=secret_name, namespace=namespace)
+        # Encode the new data to compare with the existing Secret
+        encoded_data = {k: base64.b64encode(v.encode()).decode() for k, v in data.items()}
+        
+        # Check if the existing secret's data matches the new data
+        if existing_secret.data == encoded_data:
+            logger.info("No update needed for Secret: {}".format(secret_name))
+            return False
+        else:
+            # Data is different, update the Secret
+            api_response = v1.replace_namespaced_secret(name=secret_name, namespace=namespace, body=secret)
+            logger.info("Secret updated. Name: {}".format(api_response.metadata.name))
+            return True
+
+    except ApiException as e:
+        if e.status == 404:  # Secret not found, create it
+            try:
+                api_response = v1.create_namespaced_secret(namespace=namespace, body=secret)
+                logger.info("Secret created. Name: {}".format(api_response.metadata.name))
+                return {'trigger_restart':  True}
+            except ApiException as e:
+                logger.error("Exception when creating Secret: {}".format(e))
+                raise ValueError
+        else:
+            logger.error("Failed to get or create Secret: {}".format(e))
+            raise ValueError
+        
+
+def get_kubernetes_secret(secret_name, namespace):
+    # Assuming that the Kubernetes configuration is already set
+    v1 = client.CoreV1Api()
+    try:
+        secret = v1.read_namespaced_secret(secret_name, namespace)
+        # Decoding the base64 encoded data
+        decoded_data = {key: base64.b64decode(value).decode('utf-8') for key, value in secret.data.items()}
+        return decoded_data
+    except ApiException as e:
+        logger.error(f"Failed to get secret: {e}")
+        return None
+    
+
+def restart_deployment(deployment, namespace):
+    """
+    Trigger a rollout restart of a deployment in a specified namespace.
+
+    :param name: The name of the deployment.
+    :param namespace: The namespace of the deployment.
+    """ 
+
+    v1 = client.AppsV1Api()
+    body = {
+        'spec': {
+            'template': {
+                'metadata': {
+                    'annotations': {
+                        'kubectl.kubernetes.io/restartedAt': datetime.datetime.utcnow().isoformat()
+                    }
+                }
+            }
+        }
+    }
+    try:
+        api_response = v1.patch_namespaced_deployment(name=deployment, namespace=namespace, body=body)
+        logger.info(f"Deployment restarted. Name: {api_response.metadata.name}")
+    except ApiException as e:
+        logger.error(f"Exception when restarting deployment: {e}")
+
+
+def has_mismatch_image_digest(repo_digest, label_selector, namespace):
+    """
+    Check all pods in the given namespace and matching the label selector for any
+    mismatch between the latest image digest and the current image digest.
+    
+    parameters:
+    - namespace: The namespace to search for pods.
+    - label_selector: The label selector to identify the relevant pods.
+    - repo_digest: The latest image digest to compare against.
+    
+    Returns:
+    - True if any pod is found with an image digest mismatch.
+    - False if all pods match the latest image digest.
+    """
+    core_v1_api         = client.CoreV1Api()
+
+    # Fetch pods based on namespace and label selector
+    pods = core_v1_api.list_namespaced_pod(namespace, label_selector=label_selector)
+    
+    # Iterate over pods and their containers
+    for pod in pods.items:
+        for container_status in pod.status.container_statuses:
+            current_image_id = container_status.image_id
+            # Check for digest mismatch
+            if current_image_id.split('@')[-1] != repo_digest:
+                logger.info(f"Mismatch found in pod: {pod.metadata.name}, container: {container_status.name}")
+                logger.info(f"Repo digest: {repo_digest}")
+                logger.info(f"Curr digest: {current_image_id.split('@')[-1]}")
+                return True
+    
+    logger.info("Images are in-sync.")
+    logger.info(f"Repo digest: {repo_digest}")
+    logger.info(f"Curr digest: {current_image_id.split('@')[-1]}")
+    return False

kube_watch/modules/logic/actions.py

@@ -0,0 +1,26 @@
+import subprocess
+import os
+from prefect import get_run_logger
+logger = get_run_logger()
+
+def run_standalone_script(package_name, package_run, package_exec):
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    # script_path = os.path.join(script_dir, package_name.replace('.', os.sep))
+    target_dir = os.path.join(script_dir, os.pardir, os.pardir, *package_name.split('.'))
+
+    # Change the current working directory to the script directory
+    full_command = f"{package_run} {os.path.join(target_dir, package_exec)}"
+
+    # Execute the command
+    try:
+        result = subprocess.run(full_command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if result.stdout:
+            logger.info(result.stdout)
+        if result.stderr:
+            logger.error(result.stderr)
+        # logger.info(f"Output: {result.stdout}")
+        result.check_returncode()
+    except subprocess.CalledProcessError as e:
+        # All logs should have already been handled above, now just raise an exception
+        logger.error("The subprocess encountered an error: %s", e)
+        raise Exception("Subprocess failed with exit code {}".format(e.returncode))

kube_watch/modules/logic/checks.py

@@ -0,0 +1,8 @@
+
+
+def dicts_has_diff(dict_a, dict_b):
+    return dict_a != dict_b
+
+
+def remove_keys(d, keys):
+    return {k: v for k, v in d.items() if k not in keys}

kube_watch/modules/logic/load.py

@@ -0,0 +1,8 @@
+import os
+from prefect import get_run_logger
+logger = get_run_logger()
+
+def load_secrets_to_env(data):
+    for key, value in data.items():
+        os.environ[key] = value
+        # logger.info(f"ENV VAR: {key} loaded")

kube_watch/modules/logic/merge.py

@@ -0,0 +1,31 @@
+from typing import Any, List, Dict
+from kube_watch.enums.logic import Operations
+
+
+def merge_logical_outputs(inp_dict: Dict):
+    if 'operation' not in inp_dict.keys():
+        raise TypeError("Missing required parameters: 'operation'")
+    operation = inp_dict.get('operation')
+    del inp_dict['operation']
+
+    inputs = [v for k,v in inp_dict.items()]
+    return merge_logical_list(inputs, operation)
+
+
+def merge_logical_list(inp_list: List, operation: Operations):
+    if operation == Operations.OR:
+        return any(inp_list)
+    if operation == Operations.AND:
+        return all(inp_list)
+    raise ValueError("Invalid logical operation")
+
+
+def partial_dict_update(orig_data, new_data):
+    """
+    This function is used when some key value pairs in orig_data should
+    be updated from new_data.
+    """
+    for k, v in new_data.items():
+        orig_data[k] = v
+
+    return orig_data

kube_watch/modules/logic/scheduler.py

@@ -0,0 +1,74 @@
+from datetime import datetime
+from enum import Enum
+
+from prefect import get_run_logger
+logger = get_run_logger()
+
+class IntervalType(Enum):
+    MINUTES = 'minutes'
+    HOURLY = 'hourly'
+    DAILY = 'daily'
+    WEEKLY = 'weekly'
+    MONTHLY = 'monthly'
+    QUARTERLY = 'quarterly'
+    SEMIANNUAL = 'semiannual'
+    YEARLY = 'yearly'
+
+
+def should_run_task(interval_type, interval_value=None, interval_buffer=10, specific_day=None):
+    """
+    The function `should_run_task` determines whether a task should run based on the specified interval
+    type and values.
+    
+    :param interval_type: The `interval_type` parameter specifies the type of interval at which a task
+    should run. It can take on the following values:
+    :param interval_value: The `interval_value` parameter represents the specific value associated with
+    the interval type. For example, if the interval type is `DAILY`, the `interval_value` would
+    represent the specific hour at which the task should run daily. Similarly, for `WEEKLY`, it would
+    represent the specific day
+    :param interval_buffer: The `interval_buffer` parameter in the `should_run_task` function is a
+    default value set to 20 minutes. This provides an acceptable range for a task to get executed. Suitable
+    for Daily, Weekly, Monthly, etc. schedules.
+    :param specific_day: The `specific_day` parameter represents the day of the week when the task
+    should run in the case of a weekly interval. The values for `specific_day` are as follows:
+    :return: The function `should_run_task` takes in various parameters related to different interval
+    types (such as minutes, hourly, daily, weekly, monthly, quarterly, semiannual, yearly) and checks if
+    the current datetime matches the specified interval criteria.
+    """
+
+    now = datetime.now()
+    # Match the interval type
+    if interval_type == IntervalType.MINUTES.value:
+        # Runs every 'interval_value' minutes
+        return now.minute % interval_value == 0
+    
+    if interval_type == IntervalType.HOURLY.value:
+        # Runs every 'interval_value' hours on the hour
+        return now.hour % interval_value == 0 and now.minute < interval_buffer
+    
+    if interval_type == IntervalType.DAILY.value:
+        # Runs once a day at 'interval_value' hour
+        return now.hour == interval_value and now.minute < interval_buffer
+
+    if interval_type == IntervalType.WEEKLY.value:
+        # Runs once a week on 'specific_day' (0=Monday, 6=Sunday)
+        return now.weekday() == specific_day and now.hour == 0 and now.minute < interval_buffer
+    
+    if interval_type == IntervalType.MONTHLY.value:
+        # Runs on the 'interval_value' day of each month
+        return now.day == interval_value and now.hour == 0 and now.minute < interval_buffer
+    
+    if interval_type == IntervalType.QUARTERLY.value:
+        # Runs on the first day of each quarter
+        return now.month % 3 == 1 and now.day == 1 and now.hour == 0 and now.minute < interval_buffer
+    
+    if interval_type == IntervalType.SEMIANNUAL.value:
+        # Runs on the first day of the 1st and 7th month
+        return (now.month == 1 or now.month == 7) and now.day == 1 and now.hour == 0 and now.minute < interval_buffer
+    
+    if interval_type == IntervalType.YEARLY.value:
+        # Runs on the first day of the year
+        return now.month == 1 and now.day == 1 and now.hour == 0 and now.minute < interval_buffer
+
+    return False
+

kube_watch/modules/mock/mock_generator.py

@@ -0,0 +1,24 @@
+import time
+import random
+    
+def generate_number():
+    return 42
+
+def print_number(number, dummy_param, env_var_name):
+    print(f"The generated number is: {number} and the dummy_value is: {dummy_param}")
+    return number, dummy_param, env_var_name
+
+def delay(seconds):
+    time.sleep(seconds)
+
+
+def random_boolean():
+    return random.choice([True, False])
+
+def merge_bools(inp_dict):
+    list_bools = [v for k,v in inp_dict.items()]
+    return any(list_bools)
+
+def print_result(task_name, result):
+    print(f'=========== {task_name} RESULT =================')
+    print(result)

kube_watch/modules/providers/aws.py

@@ -0,0 +1,154 @@
+#========================================================================
+# This class is deprecated. Please refer to aws.py
+#========================================================================
+import boto3
+import base64
+import json
+
+from datetime            import datetime   , timezone, timedelta
+from botocore.exceptions import ClientError
+from prefect             import get_run_logger
+from kube_watch.enums.providers import AwsResources
+
+logger = get_run_logger()
+
+def create_session(aws_creds):
+    """Create a boto3 session."""
+    session = boto3.Session(
+        aws_access_key_id=aws_creds.get('data').get('access_key'),
+        aws_secret_access_key=aws_creds.get('data').get('secret_key'),
+        aws_session_token=aws_creds.get('data').get('security_token'),  # Using .get() for optional fields
+    )
+    logger.info("Created AWS Session Successfully!")
+    return session
+
+#========================================================================================
+# ECR
+#========================================================================================
+def _prepare_ecr_secret_data(username, password, auth_token, ecr_url):
+        key_url = f"https://{ecr_url}"
+        docker_config_dict = {
+            "auths": {
+                key_url: {
+                    "username": username,
+                    "password": password,
+                    "auth": auth_token,
+                }
+            }
+        }
+        return {'.dockerconfigjson': json.dumps(docker_config_dict)}
+
+def task_get_access_token(session, resource, region, base_image_url):
+
+    if resource == AwsResources.ECR:
+        ecr_client = session.client('ecr', region_name=region)
+        token_response = ecr_client.get_authorization_token()
+        decoded_token = base64.b64decode(token_response['authorizationData'][0]['authorizationToken']).decode()
+        username, password = decoded_token.split(':')
+        return _prepare_ecr_secret_data(username, password, token_response['authorizationData'][0]['authorizationToken'], base_image_url)
+        # return {'username': username, 'password': password}
+    
+    raise ValueError('Unknown resource')
+
+
+def task_get_latest_image_digest(session, resource, region, repository_name, tag):
+    """
+    Fetches the digest of the latest image from the specified ECR repository.
+    """
+    if resource == AwsResources.ECR:
+        ecr_client = session.client('ecr', region_name=region)
+        try:
+            response = ecr_client.describe_images(
+                repositoryName=repository_name,
+                imageIds = [{'imageTag': tag}],
+                filter={'tagStatus': 'TAGGED'}
+            )
+            images = response['imageDetails']
+            # Assuming 'latest' tag is used correctly, there should be only one such image
+            if images:
+                # Extracting the digest of the latest image
+                return images[0]['imageDigest']
+        except Exception as e:
+            logger.error(f"Error fetching latest image digest: {e}")
+        return None
+    
+    raise ValueError('Unknown resource')
+
+#========================================================================================
+# IAM Cred update
+#========================================================================================
+def task_rotate_iam_creds(session, user_name, old_access_key_id, old_access_key_secret, access_key_id_var_name, access_secret_key_var_name, rotate_interval):
+    iam = session.client('iam')
+    creation_date              = None
+
+    # Retrieve the specified access key
+    has_key_exist = False
+    try:
+        response = iam.list_access_keys(UserName=user_name)
+        for key in response['AccessKeyMetadata']:
+            if key['AccessKeyId'] == old_access_key_id:
+                creation_date = key['CreateDate']
+                has_key_exist = True
+                break
+    except ClientError as error:
+        logger.error(f"Error retrieving key: {error}")
+        raise Exception(f"Error retrieving key: {error}")
+
+    if not has_key_exist:
+        logger.error(f"The provided Access Key ID; {old_access_key_id} does not exist.")
+        raise  KeyError(f"The provided Access Key ID; {old_access_key_id} does not exist.")
+
+    dd, hh, mm = list(map(lambda x: int(x), rotate_interval.split(":")))
+   
+    curr_date = datetime.now(timezone.utc)
+    # Check if the key needs rotation
+    if (curr_date.weekday() == 5 and 
+        curr_date - creation_date > timedelta(days=dd,hours=hh,minutes=mm)):
+        logger.info("Key is older than rotation period, rotating now.")
+        # Delete the old key
+        delete_iam_user_key(session, user_name, old_access_key_id)
+
+        # Create a new access key
+        access_key_id, secret_access_key = create_iam_user_key(session, user_name)
+        return {access_key_id_var_name: access_key_id, access_secret_key_var_name: secret_access_key}
+        
+    else:
+        logger.info("Key rotation not necessary.")
+        return {access_key_id_var_name: old_access_key_id, access_secret_key_var_name: old_access_key_secret}
+
+
+def create_iam_user_key(session, user_name):
+    iam       = session.client('iam')
+
+    # Check if the user exists
+    try:
+        iam.get_user(UserName=user_name)
+        logger.info("User exists, proceeding to create access key.")
+    except iam.exceptions.NoSuchEntityException:
+        raise Exception(f"User '{user_name}' does not exist, cannot proceed with key creation.")
+    except ClientError as error:
+        error_code = error.response['Error']['Code']
+        if error_code == 'NoSuchEntity':
+            raise Exception(f"User '{user_name}' does not exist in AWS IAM.")
+        else:
+            raise Exception(f"An unexpected error occurred: {error.response['Error']['Message']}")
+
+    # Create access key for this user    
+    response          = iam.create_access_key(UserName=user_name)
+    access_key_id     = response['AccessKey']['AccessKeyId']
+    secret_access_key = response['AccessKey']['SecretAccessKey']
+
+    # print(access_key_id)
+    # print(secret_access_key)
+    
+    return access_key_id, secret_access_key
+
+
+def delete_iam_user_key(session, user_name, access_key_id):
+    iam               = session.client('iam')
+    try:
+        iam.delete_access_key(UserName=user_name, AccessKeyId=access_key_id)
+        logger.info(f"Old key {access_key_id} deleted successfully.")
+    except Exception as e:
+        raise Exception(f"Failed to delete old key: {e}")
+