kubernetes-watch 0.1.5__py3-none-any.whl → 0.1.9__py3-none-any.whl

kube_watch/modules/providers/vault.py

@@ -1,188 +1,188 @@
-import hvac
-import os
-from prefect import get_run_logger
-
-from kube_watch.enums.providers import Providers
-
-logger = get_run_logger()
-
-def login(url, app_role_id, secret_id, path):
-    """
-    Login to Vault, using an existing token if available, or via AppRole otherwise.
-    
-    Parameters:
-        url (str): Vault server URL.
-        app_role_id (str): AppRole ID.
-        secret_id (str): AppRole Secret ID.
-        path (str): Path where the AppRole is enabled.
-
-    Returns:
-        dict: Dictionary containing the initialized vault_client.
-    """
-    vault_client = hvac.Client(url=url)
-    
-    # Attempt to use an existing token from environment variables
-    vault_token = os.getenv('VAULT_TOKEN', None)
-    if vault_token:
-        vault_client.token = vault_token
-        # Verify if the current token is still valid
-        try:
-            if vault_client.is_authenticated():
-                logger.info("Authenticated with existing token.")
-                return vault_client
-        except hvac.exceptions.InvalidRequest as e:
-            logger.warning(f"Failed to authenticate with the existing token: {str(e)}")
-
-    # If token is not valid or not present, authenticate with AppRole
-    try:
-        vault_client.auth.approle.login(
-            role_id=app_role_id, 
-            secret_id=secret_id, 
-            mount_point=f'approle/{path}'
-        )
-        
-        # Store the new token in environment variables for subsequent use
-        os.environ['VAULT_TOKEN'] = vault_client.token
-        logger.info("Authenticated with new token and stored in environment variable.")
-
-        return vault_client
-    except hvac.exceptions.InvalidRequest as e:
-        logger.error(f"Authentication failed with provided secret_id: {str(e)}")
-        raise RuntimeError("Authentication failed: unable to log in with the provided credentials.") from e
-
-
-
-def get_secret(vault_client, secret_path, vault_mount_point):
-    """
-    Retrieve a secret from Vault
-    """
-    res = vault_client.secrets.kv.v2.read_secret_version(
-        path=secret_path,
-        mount_point=vault_mount_point,
-        raise_on_deleted_version=True
-    )
-    return res.get('data', {}).get('data')
-
-
-def update_secret(vault_client, secret_path, secret_data, vault_mount_point):
-    """
-    Update or create a secret in Vault at the specified path.
-    
-    Args:
-        vault_client: The authenticated Vault client instance.
-        secret_path (str): The path where the secret will be stored or updated in Vault.
-        secret_data (dict): The secret data to store as a dictionary.
-        vault_mount_point (str): The mount point for the KV store.
-
-    Returns:
-        bool: True if the operation was successful, False otherwise.
-    """
-    try:
-        # Writing the secret data to Vault at the specified path
-        vault_client.secrets.kv.v2.create_or_update_secret(
-            path=secret_path,
-            secret=secret_data,
-            mount_point=vault_mount_point
-        )
-        print("Secret updated successfully.")
-        return True
-    except Exception as e:
-        print(f"Failed to update secret: {e}")
-        return False
-
-def generate_provider_creds(vault_client, provider, backend_path, role_name):
-        """
-        Generate credentials for a specified provider 
-        """
-        if provider == Providers.AWS:
-            backend_path = backend_path
-            role_name = role_name
-            creds_path = f"{backend_path}/creds/{role_name}"
-            return vault_client.read(creds_path)
-
-        raise ValueError("Unknown provider")
-
-
-
-def generate_new_secret_id(vault_client, role_name, vault_path, env_var_name):
-    """
-    Generates new secret_id. Note an admin role is required for this.
-    """
-    try:
-        # Write directly to the Vault endpoint to create the secret ID with num_uses
-        # response = vault_client.write(
-        #     f"auth/approle/{vault_path}/role/{role_name}/secret-id",
-        # )
-        response = vault_client.auth.approle.generate_secret_id(
-            role_name=role_name,
-            mount_point=f'approle/{vault_path}'
-        )
-        # Check if the response contains the secret ID
-        if response and 'data' in response:
-            secret_id = response['data']['secret_id']
-            secret_id_accessor = response['data']['secret_id_accessor']
-            logger.info("Generated a new secret ID with usage buffer.")
-            return {env_var_name: secret_id, f"{env_var_name}_ACCESSOR": secret_id_accessor}
-        else:
-            logger.error("No secret ID returned in the response.")
-            raise RuntimeError("Failed to generate new secret ID: No content returned.")
-    except hvac.exceptions.InvalidRequest as e:
-        logger.error("Error generating new secret ID: %s", str(e))
-        raise RuntimeError("Failed to generate new secret ID.") from e
-    # new_secret_response = vault_client.auth.approle.generate_secret_id(
-    #     role_name=role_name,
-    #     mount_point=f'approle/{vault_path}'
-    # )
-
-    # return { env_var_name : new_secret_response['data']['secret_id'] }
-
-
-
-def delete_secret_id(vault_client, role_name, secret_id, vault_path):
-    """
-    Delete (revoke) a secret ID associated with a role in Vault.
-    
-    Parameters:
-        vault_client (hvac.Client): An authenticated Vault client.
-        role_name (str): The name of the role the secret ID is associated with.
-        secret_id (str): The secret ID to be deleted.
-        vault_path (str): The path where the AppRole is enabled.
-    """
-    try:
-        vault_client.auth.approle.destroy_secret_id(
-            mount_point=f"approle/{vault_path}",
-            role_name=role_name,
-            secret_id=secret_id
-        )
-        
-        logger.info("Secret ID successfully revoked.")
-    except hvac.exceptions.InvalidRequest as e:
-        logger.error("Failed to revoke the secret ID: %s", str(e))
-        raise RuntimeError("Failed to delete the secret ID.") from e
-    
-
-def clean_secret_ids(vault_client, role_name, secret_id_env, vault_path, has_kube_secret_updated):
-    """
-    This function removes all idle secret-ids from `role_name`, except the 
-    inputted `secret_id_env`.
-
-    Note: secret_id_env is a dictionary. The key, VAULT_SECRET_ID, has the secret_id value.
-    """
-    secret_id = secret_id_env.get("VAULT_SECRET_ID_ACCESSOR")
-    
-    if has_kube_secret_updated:
-        secret_ids_path = f'auth/approle/{vault_path}/role/{role_name}/secret-id'
-        try:
-            response = vault_client.list(secret_ids_path)
-            if 'data' in response:
-                secret_ids = response['data']['keys']
-                for idx in secret_ids:
-                    if idx != secret_id:
-                        delete_secret_id(vault_client, role_name, secret_id, vault_path)
-                        logger.info(f"Revoking idle secret id for role: {role_name}")
-            else:
-                logger.info("No secrets found at this path.")
-        except hvac.exceptions.Forbidden:
-            logger.error("Access denied. Ensure your token has the correct policies to read this path.")
-        except Exception as e:
-            logger.error(f"An error occurred: {e}")
+import hvac
+import os
+from prefect import get_run_logger
+
+from kube_watch.enums.providers import Providers
+
+logger = get_run_logger()
+
+def login(url, app_role_id, secret_id, path):
+    """
+    Login to Vault, using an existing token if available, or via AppRole otherwise.
+    
+    Parameters:
+        url (str): Vault server URL.
+        app_role_id (str): AppRole ID.
+        secret_id (str): AppRole Secret ID.
+        path (str): Path where the AppRole is enabled.
+
+    Returns:
+        dict: Dictionary containing the initialized vault_client.
+    """
+    vault_client = hvac.Client(url=url)
+    
+    # Attempt to use an existing token from environment variables
+    vault_token = os.getenv('VAULT_TOKEN', None)
+    if vault_token:
+        vault_client.token = vault_token
+        # Verify if the current token is still valid
+        try:
+            if vault_client.is_authenticated():
+                logger.info("Authenticated with existing token.")
+                return vault_client
+        except hvac.exceptions.InvalidRequest as e:
+            logger.warning(f"Failed to authenticate with the existing token: {str(e)}")
+
+    # If token is not valid or not present, authenticate with AppRole
+    try:
+        vault_client.auth.approle.login(
+            role_id=app_role_id, 
+            secret_id=secret_id, 
+            mount_point=f'approle/{path}'
+        )
+        
+        # Store the new token in environment variables for subsequent use
+        os.environ['VAULT_TOKEN'] = vault_client.token
+        logger.info("Authenticated with new token and stored in environment variable.")
+
+        return vault_client
+    except hvac.exceptions.InvalidRequest as e:
+        logger.error(f"Authentication failed with provided secret_id: {str(e)}")
+        raise RuntimeError("Authentication failed: unable to log in with the provided credentials.") from e
+
+
+
+def get_secret(vault_client, secret_path, vault_mount_point):
+    """
+    Retrieve a secret from Vault
+    """
+    res = vault_client.secrets.kv.v2.read_secret_version(
+        path=secret_path,
+        mount_point=vault_mount_point,
+        raise_on_deleted_version=True
+    )
+    return res.get('data', {}).get('data')
+
+
+def update_secret(vault_client, secret_path, secret_data, vault_mount_point):
+    """
+    Update or create a secret in Vault at the specified path.
+    
+    Args:
+        vault_client: The authenticated Vault client instance.
+        secret_path (str): The path where the secret will be stored or updated in Vault.
+        secret_data (dict): The secret data to store as a dictionary.
+        vault_mount_point (str): The mount point for the KV store.
+
+    Returns:
+        bool: True if the operation was successful, False otherwise.
+    """
+    try:
+        # Writing the secret data to Vault at the specified path
+        vault_client.secrets.kv.v2.create_or_update_secret(
+            path=secret_path,
+            secret=secret_data,
+            mount_point=vault_mount_point
+        )
+        print("Secret updated successfully.")
+        return True
+    except Exception as e:
+        print(f"Failed to update secret: {e}")
+        return False
+
+def generate_provider_creds(vault_client, provider, backend_path, role_name):
+        """
+        Generate credentials for a specified provider 
+        """
+        if provider == Providers.AWS:
+            backend_path = backend_path
+            role_name = role_name
+            creds_path = f"{backend_path}/creds/{role_name}"
+            return vault_client.read(creds_path)
+
+        raise ValueError("Unknown provider")
+
+
+
+def generate_new_secret_id(vault_client, role_name, vault_path, env_var_name):
+    """
+    Generates new secret_id. Note an admin role is required for this.
+    """
+    try:
+        # Write directly to the Vault endpoint to create the secret ID with num_uses
+        # response = vault_client.write(
+        #     f"auth/approle/{vault_path}/role/{role_name}/secret-id",
+        # )
+        response = vault_client.auth.approle.generate_secret_id(
+            role_name=role_name,
+            mount_point=f'approle/{vault_path}'
+        )
+        # Check if the response contains the secret ID
+        if response and 'data' in response:
+            secret_id = response['data']['secret_id']
+            secret_id_accessor = response['data']['secret_id_accessor']
+            logger.info("Generated a new secret ID with usage buffer.")
+            return {env_var_name: secret_id, f"{env_var_name}_ACCESSOR": secret_id_accessor}
+        else:
+            logger.error("No secret ID returned in the response.")
+            raise RuntimeError("Failed to generate new secret ID: No content returned.")
+    except hvac.exceptions.InvalidRequest as e:
+        logger.error("Error generating new secret ID: %s", str(e))
+        raise RuntimeError("Failed to generate new secret ID.") from e
+    # new_secret_response = vault_client.auth.approle.generate_secret_id(
+    #     role_name=role_name,
+    #     mount_point=f'approle/{vault_path}'
+    # )
+
+    # return { env_var_name : new_secret_response['data']['secret_id'] }
+
+
+
+def delete_secret_id(vault_client, role_name, secret_id, vault_path):
+    """
+    Delete (revoke) a secret ID associated with a role in Vault.
+    
+    Parameters:
+        vault_client (hvac.Client): An authenticated Vault client.
+        role_name (str): The name of the role the secret ID is associated with.
+        secret_id (str): The secret ID to be deleted.
+        vault_path (str): The path where the AppRole is enabled.
+    """
+    try:
+        vault_client.auth.approle.destroy_secret_id(
+            mount_point=f"approle/{vault_path}",
+            role_name=role_name,
+            secret_id=secret_id
+        )
+        
+        logger.info("Secret ID successfully revoked.")
+    except hvac.exceptions.InvalidRequest as e:
+        logger.error("Failed to revoke the secret ID: %s", str(e))
+        raise RuntimeError("Failed to delete the secret ID.") from e
+    
+
+def clean_secret_ids(vault_client, role_name, secret_id_env, vault_path, has_kube_secret_updated):
+    """
+    This function removes all idle secret-ids from `role_name`, except the 
+    inputted `secret_id_env`.
+
+    Note: secret_id_env is a dictionary. The key, VAULT_SECRET_ID, has the secret_id value.
+    """
+    secret_id = secret_id_env.get("VAULT_SECRET_ID_ACCESSOR")
+    
+    if has_kube_secret_updated:
+        secret_ids_path = f'auth/approle/{vault_path}/role/{role_name}/secret-id'
+        try:
+            response = vault_client.list(secret_ids_path)
+            if 'data' in response:
+                secret_ids = response['data']['keys']
+                for idx in secret_ids:
+                    if idx != secret_id:
+                        delete_secret_id(vault_client, role_name, secret_id, vault_path)
+                        logger.info(f"Revoking idle secret id for role: {role_name}")
+            else:
+                logger.info("No secrets found at this path.")
+        except hvac.exceptions.Forbidden:
+            logger.error("Access denied. Ensure your token has the correct policies to read this path.")
+        except Exception as e:
+            logger.error(f"An error occurred: {e}")

kube_watch/standalone/metarecogen/ckan_to_gn.py

@@ -1,132 +1,132 @@
-import requests
-# from pathlib import Path
-import os
-import sys
-
-"""
-A simple script to:
-    1. Retrieve all public records from a CKAN service
-    2. Insert CKAN records into a Geonetwork service
-
-This requires the 'iso19115' extension to be installed in CKAN and the following env. vars:
-    1. 'CKAN2GN_GN_USERNAME' geonetwork username for an account that can create records
-    2. 'CKAN2GN_GN_PASSWORD' geonetwork password for an account that can create records
-    3. 'CKAN2GN_GN_URL' geonetork service URL
-    4. 'CKAN2GN_CKAN_URL' CKAN service URL
-"""
-
-# Geonetwork username and password:
-GN_USERNAME = os.environ.get('CKAN2GN_GN_USERNAME')
-GN_PASSWORD = os.environ.get('CKAN2GN_GN_PASSWORD')
-
-# Geonetwork and CKAN server URLs
-GN_URL = os.environ.get('CKAN2GN_GN_URL')
-CKAN_URL = os.environ.get('CKAN2GN_CKAN_URL')
-
-def get_gn_xsrf_token(session):
-    """ Retrieves XSRF token from Geonetwork
-
-    :param session: requests Session object
-    :returns: XSRF as string or None upon error
-    """
-    authenticate_url = GN_URL + '/geonetwork/srv/eng/info?type=me'
-    response = session.post(authenticate_url)
-
-    # Extract XRSF token
-    xsrf_token = response.cookies.get("XSRF-TOKEN")
-    if xsrf_token:
-        return xsrf_token
-    return None
-
-def list_ckan_records():
-    """ Contacts CKAN and retrieves a list of package ids for all public records
-
-    :returns: list of package id strings or None upon error
-    """
-    session = requests.Session()
-    url_path =  'api/3/action/package_list' # Path('api') / '3' / 'action' / 'package_list'
-    url = f'{CKAN_URL}/{url_path}'
-    r = session.get(url)
-    resp = r.json()
-    if resp['success'] is False:
-        return None
-    return resp['result']
-
-def get_ckan_record(package_id):
-    """ Given a package id retrieves its record metadata
-
-    :param package_id: CKAN package_id string
-    :returns: package metadata as a dict or None upon error
-    """
-    session = requests.Session()
-    # Set up CKAN URL
-    url_path =  'api/3/action/iso19115_package_show' #  Path('api') / '3' / 'action' / 'iso19115_package_show'
-    url = f'{CKAN_URL}/{url_path}'
-    r = session.get(url, params={'format':'xml', 'id':package_id})
-    resp = r.json()
-    if resp['success'] is False:
-        return None
-    return resp['result']
-    
-
-def insert_gn_record(session, xsrf_token, xml_string):
-    """ Inserts a record into Geonetwork
-
-    :param session: requests Session object
-    :param xsrf_token: Geonetwork's XSRF token as a string
-    :param xml_string: XML to be inserted as a string
-    :returns: True or False if insert succeeded
-    """
-    # Set header for connection
-    headers = {'Accept': 'application/json',
-               'Content-Type': 'application/xml',
-               'X-XSRF-TOKEN': xsrf_token
-    }
-
-    # Set the parameters
-    # Currently 'uuidProcessing' is set to 'NOTHING' so that records that
-    # already exist are rejected by Geonetwork as duplicates
-    params = {'metadataType': 'METADATA',
-              'publishToAll': 'true',
-              'uuidProcessing': 'NOTHING',  # Available values : GENERATEUUID, NOTHING, OVERWRITE
-    }
-
-    # Send a put request to the endpoint to create record
-    response = session.put(GN_URL + '/geonetwork/srv/api/0.1/records',
-                        data=xml_string,
-                        params=params,
-                        auth=(GN_USERNAME, GN_PASSWORD),
-                        headers=headers
-    )
-    resp = response.json()
-
-    # Check if record was created in Geonetwork
-    if response.status_code == requests.codes['created'] and resp['numberOfRecordsProcessed'] == 1 and \
-            resp['numberOfRecordsWithErrors'] == 0:
-        print("Inserted")
-        return True
-    print(f"Insert failed: status code: {response.status_code}\n{resp}")
-    return False
-        
-
-if __name__ == "__main__":
-    # Check env. vars
-    if GN_USERNAME is None or GN_PASSWORD is None or GN_URL is None or CKAN_URL is None:
-        print("Please define the following env. vars:")
-        print("           'CKAN2GN_GN_USERNAME' 'CKAN2GN_GN_PASSWORD' 'CKAN2GN_GN_URL' 'CKAN2GN_CKAN_URL'")
-        sys.exit(1)
-    # Connect to server
-    session = requests.Session()
-    xsrf = get_gn_xsrf_token(session)
-    if xsrf is not None:
-        # Get records from CKAN
-        for id in list_ckan_records():
-            print(f"Inserting '{id}'")
-            xml_string = get_ckan_record(id)
-            if xml_string is not None:
-                # Insert GN record
-                insert_gn_record(session, xsrf, xml_string)
-            else:
-                print(f"Could not get record id {id} from CKAN")
-
-    
+import requests
+# from pathlib import Path
+import os
+import sys
+
+"""
+A simple script to:
+    1. Retrieve all public records from a CKAN service
+    2. Insert CKAN records into a Geonetwork service
+
+This requires the 'iso19115' extension to be installed in CKAN and the following env. vars:
+    1. 'CKAN2GN_GN_USERNAME' geonetwork username for an account that can create records
+    2. 'CKAN2GN_GN_PASSWORD' geonetwork password for an account that can create records
+    3. 'CKAN2GN_GN_URL' geonetork service URL
+    4. 'CKAN2GN_CKAN_URL' CKAN service URL
+"""
+
+# Geonetwork username and password:
+GN_USERNAME = os.environ.get('CKAN2GN_GN_USERNAME')
+GN_PASSWORD = os.environ.get('CKAN2GN_GN_PASSWORD')
+
+# Geonetwork and CKAN server URLs
+GN_URL = os.environ.get('CKAN2GN_GN_URL')
+CKAN_URL = os.environ.get('CKAN2GN_CKAN_URL')
+
+def get_gn_xsrf_token(session):
+    """ Retrieves XSRF token from Geonetwork
+
+    :param session: requests Session object
+    :returns: XSRF as string or None upon error
+    """
+    authenticate_url = GN_URL + '/geonetwork/srv/eng/info?type=me'
+    response = session.post(authenticate_url)
+
+    # Extract XRSF token
+    xsrf_token = response.cookies.get("XSRF-TOKEN")
+    if xsrf_token:
+        return xsrf_token
+    return None
+
+def list_ckan_records():
+    """ Contacts CKAN and retrieves a list of package ids for all public records
+
+    :returns: list of package id strings or None upon error
+    """
+    session = requests.Session()
+    url_path =  'api/3/action/package_list' # Path('api') / '3' / 'action' / 'package_list'
+    url = f'{CKAN_URL}/{url_path}'
+    r = session.get(url)
+    resp = r.json()
+    if resp['success'] is False:
+        return None
+    return resp['result']
+
+def get_ckan_record(package_id):
+    """ Given a package id retrieves its record metadata
+
+    :param package_id: CKAN package_id string
+    :returns: package metadata as a dict or None upon error
+    """
+    session = requests.Session()
+    # Set up CKAN URL
+    url_path =  'api/3/action/iso19115_package_show' #  Path('api') / '3' / 'action' / 'iso19115_package_show'
+    url = f'{CKAN_URL}/{url_path}'
+    r = session.get(url, params={'format':'xml', 'id':package_id})
+    resp = r.json()
+    if resp['success'] is False:
+        return None
+    return resp['result']
+    
+
+def insert_gn_record(session, xsrf_token, xml_string):
+    """ Inserts a record into Geonetwork
+
+    :param session: requests Session object
+    :param xsrf_token: Geonetwork's XSRF token as a string
+    :param xml_string: XML to be inserted as a string
+    :returns: True or False if insert succeeded
+    """
+    # Set header for connection
+    headers = {'Accept': 'application/json',
+               'Content-Type': 'application/xml',
+               'X-XSRF-TOKEN': xsrf_token
+    }
+
+    # Set the parameters
+    # Currently 'uuidProcessing' is set to 'NOTHING' so that records that
+    # already exist are rejected by Geonetwork as duplicates
+    params = {'metadataType': 'METADATA',
+              'publishToAll': 'true',
+              'uuidProcessing': 'NOTHING',  # Available values : GENERATEUUID, NOTHING, OVERWRITE
+    }
+
+    # Send a put request to the endpoint to create record
+    response = session.put(GN_URL + '/geonetwork/srv/api/0.1/records',
+                        data=xml_string,
+                        params=params,
+                        auth=(GN_USERNAME, GN_PASSWORD),
+                        headers=headers
+    )
+    resp = response.json()
+
+    # Check if record was created in Geonetwork
+    if response.status_code == requests.codes['created'] and resp['numberOfRecordsProcessed'] == 1 and \
+            resp['numberOfRecordsWithErrors'] == 0:
+        print("Inserted")
+        return True
+    print(f"Insert failed: status code: {response.status_code}\n{resp}")
+    return False
+        
+
+if __name__ == "__main__":
+    # Check env. vars
+    if GN_USERNAME is None or GN_PASSWORD is None or GN_URL is None or CKAN_URL is None:
+        print("Please define the following env. vars:")
+        print("           'CKAN2GN_GN_USERNAME' 'CKAN2GN_GN_PASSWORD' 'CKAN2GN_GN_URL' 'CKAN2GN_CKAN_URL'")
+        sys.exit(1)
+    # Connect to server
+    session = requests.Session()
+    xsrf = get_gn_xsrf_token(session)
+    if xsrf is not None:
+        # Get records from CKAN
+        for id in list_ckan_records():
+            print(f"Inserting '{id}'")
+            xml_string = get_ckan_record(id)
+            if xml_string is not None:
+                # Insert GN record
+                insert_gn_record(session, xsrf, xml_string)
+            else:
+                print(f"Could not get record id {id} from CKAN")
+
+    

kube_watch/watch/__init__.py

@@ -1 +1 @@
-from .workflow import single_run_workflow, batch_run_workflow
+from .workflow import single_run_workflow_async, batch_run_workflow, single_run_workflow