kubernetes-watch 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

kube_watch/enums/workflow.py

@@ -3,6 +3,7 @@ from enum import Enum
 class ParameterType(str, Enum):
     STATIC = 'static'
     FROM_ENV = 'env'
+    FROM_FLOW = 'flow'
 
 
 class TaskRunners(str, Enum):

kube_watch/models/workflow.py

@@ -25,10 +25,14 @@ class Condition(CamelModel):
     operation: Optional[Operations] = Operations.AND
 
 class Task(CamelModel):
+    """
+    :param plugin_path: define if referring to an external module outside the library.
+    """
     module: str
     task: str
     name: str
-    inputsArgType: Optional[TaskInputsType] = TaskInputsType.ARG
+    plugin_path: Optional[str] = ""
+    inputsArgType: Optional[TaskInputsType] = TaskInputsType.ARG  # @TODO refactor inputsArgType to inputs_arg_type
     inputs: Optional[Inputs] = None
     dependency: Optional[List[Dependency]]  = None
     conditional: Optional[Condition]  = None
@@ -37,6 +41,7 @@ class Task(CamelModel):
 class WorkflowConfig(CamelModel):
     name: str
     runner: TaskRunners = TaskRunners.CONCURRENT
+    parameters: Optional[List[Parameter]] = []
     tasks: List[Task]
 
 class WorkflowOutput(CamelModel):

kube_watch/modules/logic/actions.py

@@ -1,5 +1,6 @@
 import subprocess
 import os
+from typing import List
 from prefect import get_run_logger
 logger = get_run_logger()
 
@@ -8,7 +9,6 @@ def run_standalone_script(package_name, package_run, package_exec):
     # script_path = os.path.join(script_dir, package_name.replace('.', os.sep))
     target_dir = os.path.join(script_dir, os.pardir, os.pardir, *package_name.split('.'))
 
-    # Change the current working directory to the script directory
     full_command = f"{package_run} {os.path.join(target_dir, package_exec)}"
 
     # Execute the command
@@ -23,4 +23,34 @@ def run_standalone_script(package_name, package_run, package_exec):
     except subprocess.CalledProcessError as e:
         # All logs should have already been handled above, now just raise an exception
         logger.error("The subprocess encountered an error: %s", e)
-        raise Exception("Subprocess failed with exit code {}".format(e.returncode))
+        raise Exception("Subprocess failed with exit code {}".format(e.returncode))
+    
+
+def run_standalone_script_modified(base_path: str, package_name: str, package_run_cmds: List[str]):
+    # Construct the absolute path to the target directory
+    target_dir = os.path.join(base_path, *package_name.split('.'))
+
+    commands = [f"cd {target_dir}"] + package_run_cmds
+    full_command = " && ".join(commands)
+
+    # full_command = f"cd {target_dir} && {package_run_cmd}"
+
+    # Build the full command to execute
+    # full_command = f"{package_run} {os.path.join(target_dir, package_exec)}"
+
+    # print(full_command)
+
+    # Execute the command
+    try:
+        result = subprocess.run(full_command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if result.stdout:
+            logger.info(result.stdout)
+        if result.stderr:
+            logger.error(result.stderr)
+        result.check_returncode()
+    except subprocess.CalledProcessError as e:
+        logger.error("Command failed with exit code %s", e.returncode)
+        logger.error("Output:\n%s", e.stdout)
+        logger.error("Errors:\n%s", e.stderr)
+        raise Exception(f"Subprocess failed with exit code {e.returncode}. Check logs for more details.")
+        # raise Exception(f"Subprocess failed with exit code {e.returncode}")

kube_watch/modules/mock/mock_generator.py

@@ -1,13 +1,42 @@
 import time
 import random
+from prefect import runtime
     
 def generate_number():
     return 42
 
+
+def mock_dict_data():
+    return {
+        "key1": "value1",
+        "key2": "value2",
+        "key3": {
+            "k1": 1, "k2": 2, "k3": [1, 3, "ali"]
+        }
+    }
+
+def print_input(params):
+    print(params)
+
 def print_number(number, dummy_param, env_var_name):
     print(f"The generated number is: {number} and the dummy_value is: {dummy_param}")
     return number, dummy_param, env_var_name
 
+
+def print_flow_parameters():
+    assert runtime.flow_run.parameters.get("WORK_DIR") is not None
+    assert runtime.flow_run.parameters.get("MODULE_PATH") is not None
+    print(runtime.flow_run.parameters.get("WORK_DIR"))
+    print(runtime.flow_run.parameters.get("MODULE_PATH"))
+
+def print_from_flow_parameters(work_dir):
+    """
+    work_dir is provided via flow parameters
+    """
+    assert work_dir != ''
+    print(work_dir)
+
+
 def delay(seconds):
     time.sleep(seconds)
 

kube_watch/modules/providers/aws.py

@@ -4,7 +4,6 @@
 import boto3
 import base64
 import json
-
 from datetime            import datetime   , timezone, timedelta
 from botocore.exceptions import ClientError
 from prefect             import get_run_logger
@@ -77,7 +76,17 @@ def task_get_latest_image_digest(session, resource, region, repository_name, tag
 #========================================================================================
 # IAM Cred update
 #========================================================================================
-def task_rotate_iam_creds(session, user_name, old_access_key_id, old_access_key_secret, access_key_id_var_name, access_secret_key_var_name, rotate_interval):
+def task_rotate_iam_creds(
+        session,
+        user_name,
+        old_access_key_id,
+        old_access_key_secret,
+        access_key_id_var_name,
+        access_secret_key_var_name,
+        rotate_interval,
+        require_smtp_conversion = False,
+        ses_region = "ap-southeast-2"
+    ):
     iam = session.client('iam')
     creation_date              = None
 
@@ -102,14 +111,15 @@ def task_rotate_iam_creds(session, user_name, old_access_key_id, old_access_key_
    
     curr_date = datetime.now(timezone.utc)
     # Check if the key needs rotation
-    if (curr_date.weekday() == 5 and 
-        curr_date - creation_date > timedelta(days=dd,hours=hh,minutes=mm)):
+    if (curr_date - creation_date > timedelta(days=dd,hours=hh,minutes=mm)):
         logger.info("Key is older than rotation period, rotating now.")
         # Delete the old key
         delete_iam_user_key(session, user_name, old_access_key_id)
 
         # Create a new access key
         access_key_id, secret_access_key = create_iam_user_key(session, user_name)
+        if require_smtp_conversion:
+            secret_access_key = convert_to_smtp_password(secret_access_key, ses_region)
         return {access_key_id_var_name: access_key_id, access_secret_key_var_name: secret_access_key}
         
     else:
@@ -117,9 +127,9 @@ def task_rotate_iam_creds(session, user_name, old_access_key_id, old_access_key_
         return {access_key_id_var_name: old_access_key_id, access_secret_key_var_name: old_access_key_secret}
 
 
+
 def create_iam_user_key(session, user_name):
     iam       = session.client('iam')
-
     # Check if the user exists
     try:
         iam.get_user(UserName=user_name)
@@ -152,3 +162,50 @@ def delete_iam_user_key(session, user_name, access_key_id):
     except Exception as e:
         raise Exception(f"Failed to delete old key: {e}")
 
+    
+
+def convert_to_smtp_password(secret_access_key, region):
+    """Convert IAM Secret Key to SMTP Password."""
+    import hmac
+    import hashlib
+    import base64
+
+    SMTP_REGIONS = [
+        'us-east-2',       # US East (Ohio)
+        'us-east-1',       # US East (N. Virginia)
+        'us-west-2',       # US West (Oregon)
+        'ap-south-1',      # Asia Pacific (Mumbai)
+        'ap-northeast-2',  # Asia Pacific (Seoul)
+        'ap-southeast-1',  # Asia Pacific (Singapore)
+        'ap-southeast-2',  # Asia Pacific (Sydney)
+        'ap-northeast-1',  # Asia Pacific (Tokyo)
+        'ca-central-1',    # Canada (Central)
+        'eu-central-1',    # Europe (Frankfurt)
+        'eu-west-1',       # Europe (Ireland)
+        'eu-west-2',       # Europe (London)
+        'sa-east-1',       # South America (Sao Paulo)
+        'us-gov-west-1',   # AWS GovCloud (US)
+    ]
+
+    # These values are required to calculate the signature. Do not change them.
+    DATE = "11111111"
+    SERVICE = "ses"
+    MESSAGE = "SendRawEmail"
+    TERMINAL = "aws4_request"
+    VERSION = 0x04
+
+
+    def sign(key, msg):
+        return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
+    
+    if region not in SMTP_REGIONS:
+        raise ValueError(f"The {region} Region doesn't have an SMTP endpoint.")
+
+    signature = sign(("AWS4" + secret_access_key).encode('utf-8'), DATE)
+    signature = sign(signature, region)
+    signature = sign(signature, SERVICE)
+    signature = sign(signature, TERMINAL)
+    signature = sign(signature, MESSAGE)
+    signature_and_version = bytes([VERSION]) + signature
+    smtp_password = base64.b64encode(signature_and_version)
+    return smtp_password.decode('utf-8')

kube_watch/watch/helpers.py

@@ -1,7 +1,7 @@
 from prefect import task
-import functools
-import asyncio
+import sys
 from prefect.task_runners import ConcurrentTaskRunner, SequentialTaskRunner
+from prefect import runtime
 # from prefect_dask.task_runners import DaskTaskRunner
 from typing import Dict, List
 import yaml
@@ -51,28 +51,46 @@ def func_task(name="default_task_name", task_input_type: TaskInputsType = TaskIn
 
 
 
-def get_task_function(module_name, task_name):
-    # module = importlib.import_module(f"sparrow_watch.modules.{module_name}")
-    # klass = getattr(module, class_name)
-    # return getattr(klass, task_name)
+def get_task_function(module_name, task_name, plugin_path=None):
     """
     Fetch a function directly from a specified module.
     
     Args:
         module_name (str): The name of the module to import the function from. e.g. providers.aws
         task_name (str): The name of the function to fetch from the module.
+        plugin_path (ster): define for external modules
         
     Returns:
         function: The function object fetched from the module.
     """
-    module = importlib.import_module(f"kube_watch.modules.{module_name}")
-    return getattr(module, task_name)
+    try:
+        if plugin_path:
+            # Temporarily prepend the plugin path to sys.path to find the module
+            module_path = os.path.join(plugin_path, *module_name.split('.')) + '.py'
+            module_spec = importlib.util.spec_from_file_location(module_name, module_path)
+            module = importlib.util.module_from_spec(module_spec)
+            module_spec.loader.exec_module(module)
+        else:
+            # Standard import from the internal library path
+            module = importlib.import_module(f"kube_watch.modules.{module_name}")
+        
+        return getattr(module, task_name)
+    except ImportError as e:
+        raise ImportError(f"Unable to import module '{module_name}': {e}")
+    except AttributeError as e:
+        raise AttributeError(f"The module '{module_name}' does not have a function named '{task_name}': {e}")
+    finally:
+        if plugin_path:
+            # Remove the plugin path from sys.path after importing
+            sys.path.pop(0)  # Using pop(0) is safer in the context of insert(0, plugin_path)
 
 
 
 def resolve_parameter_value(param):
     if param.type == ParameterType.FROM_ENV:
         return os.getenv(param.value, '')  # Default to empty string if env var is not set
+    if param.type == ParameterType.FROM_FLOW:
+        return runtime.flow_run.parameters.get(param.value, '')
     return param.value
 
 def prepare_task_inputs(parameters):

kube_watch/watch/workflow.py

@@ -1,4 +1,4 @@
-from prefect import flow,  get_run_logger
+from prefect import flow,  get_run_logger, runtime
 import asyncio
 from typing import List
 import secrets
@@ -20,10 +20,14 @@ def create_flow_based_on_config(yaml_file, run_async=True):
     async def dynamic_workflow():
         logger = get_run_logger()
         tasks = {}
+
+        for param in workflow_config.parameters:      
+            runtime.flow_run.parameters[param.name] = param.value
+
         logger.info(f"Starting flow: {flow_name}")
         for task_data in workflow_config.tasks:
             task_name   = task_data.name
-            func        = helpers.get_task_function(task_data.module, task_data.task)
+            func        = helpers.get_task_function(task_data.module, task_data.task, task_data.plugin_path)
             task_inputs = helpers.prepare_task_inputs(task_data.inputs.parameters) if task_data.inputs else {}
 
             condition_result = True
@@ -38,17 +42,6 @@ def create_flow_based_on_config(yaml_file, run_async=True):
                 task_future = helpers.submit_task(task_name, task_data, task_inputs, func)
                 tasks[task_data.name] = task_future
             
-            # if task_data.dependency:
-            #     task_inputs = helpers.prepare_task_inputs_from_dep(task_data, task_inputs, tasks)
-        
-            # if task_data.conditional:
-            #     condition_result  = helpers.resolve_conditional(task_data, tasks)
-            #     if condition_result:
-            #         task_future = helpers.submit_task(task_name, task_data, task_inputs, func)
-            #         tasks[task_data.name] = task_future
-            # else:
-            #     task_future = helpers.submit_task(task_name, task_data, task_inputs, func)
-            #     tasks[task_data.name] = task_future
 
         return tasks
     return dynamic_workflow

{kubernetes_watch-0.1.2.dist-info → kubernetes_watch-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kubernetes-watch
-Version: 0.1.2
+Version: 0.1.3
 Summary: 
 Author: bmotevalli
 Author-email: b.motevalli@gmail.com
@@ -9,14 +9,14 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: GitPython (==3.1.43)
-Requires-Dist: PyYAML (==6.0.1)
-Requires-Dist: boto3 (==1.34.68)
-Requires-Dist: humps (==0.2.2)
-Requires-Dist: hvac (==2.1.0)
-Requires-Dist: kubernetes (==29.0.0)
-Requires-Dist: prefect (==2.18.0)
-Requires-Dist: requests (==2.32.3)
+Requires-Dist: GitPython (>=3.1.43,<4.0.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: boto3 (>=1.34.68,<2.0.0)
+Requires-Dist: humps (>=0.2.2,<0.3.0)
+Requires-Dist: hvac (>=2.1.0,<3.0.0)
+Requires-Dist: kubernetes (>=29.0.0,<30.0.0)
+Requires-Dist: prefect (>=2.18.0,<3.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
 Description-Content-Type: text/markdown
 
 # kube_watch
@@ -26,10 +26,16 @@ Description-Content-Type: text/markdown
 - `poetry install`
 - `poetry shell`
 
-# To install package to your environment:
+# To install package to your environment locally:
 
 python setup.py install
 
+# To publish
+
+`poetry config pypi-token.pypi your-api-token`
+`poetry build`
+`poetry publish`
+
 
 # Description
 The kube_watch library is build on top of <a href='https://docs.prefect.io/latest/'>Prefect</a>. The library is designed to define workflows in a declaritive and flexible fashion. Originally, workflows in Prefect are defined via decorators such as @flow and @task. In kube_watch, workflows can be defined in a declaritive form via yaml files. The library is mainly focused on running scheduled workflows in kubernetes environment. However, it can easily be extended to be used for any purpose requiring a workflow. The workflow manifest has the following generic structure:

{kubernetes_watch-0.1.2.dist-info → kubernetes_watch-0.1.3.dist-info}/RECORD

@@ -3,32 +3,31 @@ kube_watch/enums/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 kube_watch/enums/kube.py,sha256=z6ceQwHV9-LB1bMitaMeX8kBXt2GCVdVDFAwc6DQffo,94
 kube_watch/enums/logic.py,sha256=Dzh24fZpYahIt0YWpXe1_4FIoJNlwwgYOCnwEAjo8Uk,154
 kube_watch/enums/providers.py,sha256=nMX-hXqhgLJMFmC5nmMy8Ajnr7tiya3B5NWn57EMcxk,248
-kube_watch/enums/workflow.py,sha256=yMViHxigDfft7Qmsg3ou7x-vmMbw2IopbMkeNtHhbMI,321
+kube_watch/enums/workflow.py,sha256=W4EI98Kwh2Ptbzn4KgRMtTnulyKYxICRPgxo4g7wTjU,345
 kube_watch/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kube_watch/models/common.py,sha256=mECTcFVLEitlN38l8tqP-BI1kwYev3Jxq0kDSH5pE0o,543
-kube_watch/models/workflow.py,sha256=S1c5cWAXJYnnRUHglE9JO61cfoWw4Q26o1wfZ2SKdsY,1452
+kube_watch/models/workflow.py,sha256=WE-ArxyaJfmze6gRvmwHYKQu7TpQBzxy88szqJO4Xxc,1694
 kube_watch/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kube_watch/modules/clusters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kube_watch/modules/clusters/kube.py,sha256=Fje6-vUA1KQ4x8T6cUYJT_eVwUYw-dR71h95ixSLqLM,7767
-kube_watch/modules/logic/actions.py,sha256=Nf5-pzKobcd-CYNWhaz-j6wucAE6cb2nKqfNzaM26Tw,1223
+kube_watch/modules/logic/actions.py,sha256=lt7OkSw6m2ZQe2SfENUsjZeD8vdpvfoE4laEkv9veEA,2471
 kube_watch/modules/logic/checks.py,sha256=CFIMVURKJP5Y3mByyJkFCrJBlVUjTG2XixiwoRquXN4,157
 kube_watch/modules/logic/load.py,sha256=5DKi3nAFMcoNQL504JaDc-_REJSc6GyxeXLWQwNl0BM,227
 kube_watch/modules/logic/merge.py,sha256=vwc2TwcGU-vH5W0bFXzAzOMHt36ksdS4if1c4IbTeXs,926
 kube_watch/modules/logic/scheduler.py,sha256=-p5qh3FnEQ1jlkaY0Lrj9U-vau1b07NYAXBP6M09yoU,3517
 kube_watch/modules/logic/trasnform.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kube_watch/modules/mock/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kube_watch/modules/mock/mock_generator.py,sha256=V7u5e_j_XIzFnv4sqhd44wBPuJIjd8ZeH9znfqxtIyE,609
+kube_watch/modules/mock/mock_generator.py,sha256=j8UfcJeA9giEEyqH9Sf3RGtlMfGO13NbWMZ80dj4UtE,1315
 kube_watch/modules/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kube_watch/modules/providers/aws.py,sha256=N3dmImU0_yIJJS0JrW8FqJ-66_jGip0QBfb7-4VDKTA,6606
+kube_watch/modules/providers/aws.py,sha256=1iyekXCeSu0owoS46VXFPp7iRy9U5bUMgrTD8BldpPM,8585
 kube_watch/modules/providers/git.py,sha256=h3rcn1FhU82nF52Ol9YHyFk4cvPxxaz_AxHnip8OXPY,1183
 kube_watch/modules/providers/github.py,sha256=WCpZIKHr4U0a4El1leXkaCv1jznf9ob5xHVeTNSpNG0,5338
 kube_watch/modules/providers/vault.py,sha256=eGQbjrVLlHzOPW8SqRWn6S6j8WsQBvvG7jOhzJKEh1o,3983
 kube_watch/standalone/metarecogen/ckan_to_gn.py,sha256=FBiv6McWh4hqV6Bz08zGLzEIe4v1-D3FawjBKYbV7Ms,4767
-kube_watch/standalone/metarecogen/Dockerfile,sha256=5EBb8oUnBwlRcLT4Lw2E659ONj4LYk_HFF4lZfjB2DM,357
 kube_watch/watch/__init__.py,sha256=6Ay9P_Ws7rP7ZaIrFRZtp_1uwVK4ZDmkkNhFyqPNQIU,61
-kube_watch/watch/helpers.py,sha256=MlsPuUHIYfKHB8GZf2yEz779hQwen_s3xBOp9gQu6oI,4377
-kube_watch/watch/workflow.py,sha256=7AODfNDx9oc1lAuPfF4muATs-6eTZZk6WfD5yN20tGo,4688
-kubernetes_watch-0.1.2.dist-info/LICENSE,sha256=StyinJRmy--Pc2vQbRToZSN4sjSVg3zccMFrktVcrEw,1096
-kubernetes_watch-0.1.2.dist-info/METADATA,sha256=r3b4BplzbwQTuN1iSEbl26mTHoaxDhfFSd6GinbQt-w,4810
-kubernetes_watch-0.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-kubernetes_watch-0.1.2.dist-info/RECORD,,
+kube_watch/watch/helpers.py,sha256=JRXGIoDKcsBnGirlr8Et0Qaz8aPsKz4WsmFFwcEmml8,5373
+kube_watch/watch/workflow.py,sha256=h0b_P_kfiPxqTFHZ6o2HkDkNaUBOwv1DKJnwEMMVXaI,4203
+kubernetes_watch-0.1.3.dist-info/LICENSE,sha256=StyinJRmy--Pc2vQbRToZSN4sjSVg3zccMFrktVcrEw,1096
+kubernetes_watch-0.1.3.dist-info/METADATA,sha256=VqRB-sKEaMUf-hh4kuADHA9Eau8o9NnKloEGDCy5XNM,4969
+kubernetes_watch-0.1.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+kubernetes_watch-0.1.3.dist-info/RECORD,,

kube_watch/standalone/metarecogen/Dockerfile

@@ -1,16 +0,0 @@
-FROM python:3.11-slim
-# FROM python:3
-# Simple Dockerfile used to run 'ckan_to_gn.py' script
-
-# Install pip and requests library
-RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir requests
-
-# Set work dir
-WORKDIR /usr/app/src
-
-# Copy python script
-COPY ckan_to_gn.py ./
-
-# Run it
-CMD ["python3", "ckan_to_gn.py"]