kubernetes-watch 0.1.4__py3-none-any.whl → 0.1.8__py3-none-any.whl

kube_watch/standalone/metarecogen/ckan_to_gn.py

@@ -1,132 +1,132 @@
-import requests
-# from pathlib import Path
-import os
-import sys
-
-"""
-A simple script to:
-    1. Retrieve all public records from a CKAN service
-    2. Insert CKAN records into a Geonetwork service
-
-This requires the 'iso19115' extension to be installed in CKAN and the following env. vars:
-    1. 'CKAN2GN_GN_USERNAME' geonetwork username for an account that can create records
-    2. 'CKAN2GN_GN_PASSWORD' geonetwork password for an account that can create records
-    3. 'CKAN2GN_GN_URL' geonetork service URL
-    4. 'CKAN2GN_CKAN_URL' CKAN service URL
-"""
-
-# Geonetwork username and password:
-GN_USERNAME = os.environ.get('CKAN2GN_GN_USERNAME')
-GN_PASSWORD = os.environ.get('CKAN2GN_GN_PASSWORD')
-
-# Geonetwork and CKAN server URLs
-GN_URL = os.environ.get('CKAN2GN_GN_URL')
-CKAN_URL = os.environ.get('CKAN2GN_CKAN_URL')
-
-def get_gn_xsrf_token(session):
-    """ Retrieves XSRF token from Geonetwork
-
-    :param session: requests Session object
-    :returns: XSRF as string or None upon error
-    """
-    authenticate_url = GN_URL + '/geonetwork/srv/eng/info?type=me'
-    response = session.post(authenticate_url)
-
-    # Extract XRSF token
-    xsrf_token = response.cookies.get("XSRF-TOKEN")
-    if xsrf_token:
-        return xsrf_token
-    return None
-
-def list_ckan_records():
-    """ Contacts CKAN and retrieves a list of package ids for all public records
-
-    :returns: list of package id strings or None upon error
-    """
-    session = requests.Session()
-    url_path =  'api/3/action/package_list' # Path('api') / '3' / 'action' / 'package_list'
-    url = f'{CKAN_URL}/{url_path}'
-    r = session.get(url)
-    resp = r.json()
-    if resp['success'] is False:
-        return None
-    return resp['result']
-
-def get_ckan_record(package_id):
-    """ Given a package id retrieves its record metadata
-
-    :param package_id: CKAN package_id string
-    :returns: package metadata as a dict or None upon error
-    """
-    session = requests.Session()
-    # Set up CKAN URL
-    url_path =  'api/3/action/iso19115_package_show' #  Path('api') / '3' / 'action' / 'iso19115_package_show'
-    url = f'{CKAN_URL}/{url_path}'
-    r = session.get(url, params={'format':'xml', 'id':package_id})
-    resp = r.json()
-    if resp['success'] is False:
-        return None
-    return resp['result']
-    
-
-def insert_gn_record(session, xsrf_token, xml_string):
-    """ Inserts a record into Geonetwork
-
-    :param session: requests Session object
-    :param xsrf_token: Geonetwork's XSRF token as a string
-    :param xml_string: XML to be inserted as a string
-    :returns: True or False if insert succeeded
-    """
-    # Set header for connection
-    headers = {'Accept': 'application/json',
-               'Content-Type': 'application/xml',
-               'X-XSRF-TOKEN': xsrf_token
-    }
-
-    # Set the parameters
-    # Currently 'uuidProcessing' is set to 'NOTHING' so that records that
-    # already exist are rejected by Geonetwork as duplicates
-    params = {'metadataType': 'METADATA',
-              'publishToAll': 'true',
-              'uuidProcessing': 'NOTHING',  # Available values : GENERATEUUID, NOTHING, OVERWRITE
-    }
-
-    # Send a put request to the endpoint to create record
-    response = session.put(GN_URL + '/geonetwork/srv/api/0.1/records',
-                        data=xml_string,
-                        params=params,
-                        auth=(GN_USERNAME, GN_PASSWORD),
-                        headers=headers
-    )
-    resp = response.json()
-
-    # Check if record was created in Geonetwork
-    if response.status_code == requests.codes['created'] and resp['numberOfRecordsProcessed'] == 1 and \
-            resp['numberOfRecordsWithErrors'] == 0:
-        print("Inserted")
-        return True
-    print(f"Insert failed: status code: {response.status_code}\n{resp}")
-    return False
-        
-
-if __name__ == "__main__":
-    # Check env. vars
-    if GN_USERNAME is None or GN_PASSWORD is None or GN_URL is None or CKAN_URL is None:
-        print("Please define the following env. vars:")
-        print("           'CKAN2GN_GN_USERNAME' 'CKAN2GN_GN_PASSWORD' 'CKAN2GN_GN_URL' 'CKAN2GN_CKAN_URL'")
-        sys.exit(1)
-    # Connect to server
-    session = requests.Session()
-    xsrf = get_gn_xsrf_token(session)
-    if xsrf is not None:
-        # Get records from CKAN
-        for id in list_ckan_records():
-            print(f"Inserting '{id}'")
-            xml_string = get_ckan_record(id)
-            if xml_string is not None:
-                # Insert GN record
-                insert_gn_record(session, xsrf, xml_string)
-            else:
-                print(f"Could not get record id {id} from CKAN")
-
-    
+import requests
+# from pathlib import Path
+import os
+import sys
+
+"""
+A simple script to:
+    1. Retrieve all public records from a CKAN service
+    2. Insert CKAN records into a Geonetwork service
+
+This requires the 'iso19115' extension to be installed in CKAN and the following env. vars:
+    1. 'CKAN2GN_GN_USERNAME' geonetwork username for an account that can create records
+    2. 'CKAN2GN_GN_PASSWORD' geonetwork password for an account that can create records
+    3. 'CKAN2GN_GN_URL' geonetork service URL
+    4. 'CKAN2GN_CKAN_URL' CKAN service URL
+"""
+
+# Geonetwork username and password:
+GN_USERNAME = os.environ.get('CKAN2GN_GN_USERNAME')
+GN_PASSWORD = os.environ.get('CKAN2GN_GN_PASSWORD')
+
+# Geonetwork and CKAN server URLs
+GN_URL = os.environ.get('CKAN2GN_GN_URL')
+CKAN_URL = os.environ.get('CKAN2GN_CKAN_URL')
+
+def get_gn_xsrf_token(session):
+    """ Retrieves XSRF token from Geonetwork
+
+    :param session: requests Session object
+    :returns: XSRF as string or None upon error
+    """
+    authenticate_url = GN_URL + '/geonetwork/srv/eng/info?type=me'
+    response = session.post(authenticate_url)
+
+    # Extract XRSF token
+    xsrf_token = response.cookies.get("XSRF-TOKEN")
+    if xsrf_token:
+        return xsrf_token
+    return None
+
+def list_ckan_records():
+    """ Contacts CKAN and retrieves a list of package ids for all public records
+
+    :returns: list of package id strings or None upon error
+    """
+    session = requests.Session()
+    url_path =  'api/3/action/package_list' # Path('api') / '3' / 'action' / 'package_list'
+    url = f'{CKAN_URL}/{url_path}'
+    r = session.get(url)
+    resp = r.json()
+    if resp['success'] is False:
+        return None
+    return resp['result']
+
+def get_ckan_record(package_id):
+    """ Given a package id retrieves its record metadata
+
+    :param package_id: CKAN package_id string
+    :returns: package metadata as a dict or None upon error
+    """
+    session = requests.Session()
+    # Set up CKAN URL
+    url_path =  'api/3/action/iso19115_package_show' #  Path('api') / '3' / 'action' / 'iso19115_package_show'
+    url = f'{CKAN_URL}/{url_path}'
+    r = session.get(url, params={'format':'xml', 'id':package_id})
+    resp = r.json()
+    if resp['success'] is False:
+        return None
+    return resp['result']
+    
+
+def insert_gn_record(session, xsrf_token, xml_string):
+    """ Inserts a record into Geonetwork
+
+    :param session: requests Session object
+    :param xsrf_token: Geonetwork's XSRF token as a string
+    :param xml_string: XML to be inserted as a string
+    :returns: True or False if insert succeeded
+    """
+    # Set header for connection
+    headers = {'Accept': 'application/json',
+               'Content-Type': 'application/xml',
+               'X-XSRF-TOKEN': xsrf_token
+    }
+
+    # Set the parameters
+    # Currently 'uuidProcessing' is set to 'NOTHING' so that records that
+    # already exist are rejected by Geonetwork as duplicates
+    params = {'metadataType': 'METADATA',
+              'publishToAll': 'true',
+              'uuidProcessing': 'NOTHING',  # Available values : GENERATEUUID, NOTHING, OVERWRITE
+    }
+
+    # Send a put request to the endpoint to create record
+    response = session.put(GN_URL + '/geonetwork/srv/api/0.1/records',
+                        data=xml_string,
+                        params=params,
+                        auth=(GN_USERNAME, GN_PASSWORD),
+                        headers=headers
+    )
+    resp = response.json()
+
+    # Check if record was created in Geonetwork
+    if response.status_code == requests.codes['created'] and resp['numberOfRecordsProcessed'] == 1 and \
+            resp['numberOfRecordsWithErrors'] == 0:
+        print("Inserted")
+        return True
+    print(f"Insert failed: status code: {response.status_code}\n{resp}")
+    return False
+        
+
+if __name__ == "__main__":
+    # Check env. vars
+    if GN_USERNAME is None or GN_PASSWORD is None or GN_URL is None or CKAN_URL is None:
+        print("Please define the following env. vars:")
+        print("           'CKAN2GN_GN_USERNAME' 'CKAN2GN_GN_PASSWORD' 'CKAN2GN_GN_URL' 'CKAN2GN_CKAN_URL'")
+        sys.exit(1)
+    # Connect to server
+    session = requests.Session()
+    xsrf = get_gn_xsrf_token(session)
+    if xsrf is not None:
+        # Get records from CKAN
+        for id in list_ckan_records():
+            print(f"Inserting '{id}'")
+            xml_string = get_ckan_record(id)
+            if xml_string is not None:
+                # Insert GN record
+                insert_gn_record(session, xsrf, xml_string)
+            else:
+                print(f"Could not get record id {id} from CKAN")
+
+    

kube_watch/watch/__init__.py

@@ -1 +1 @@
-from .workflow import single_run_workflow, batch_run_workflow
+from .workflow import single_run_workflow_async, batch_run_workflow, single_run_workflow

kube_watch/watch/helpers.py

@@ -1,171 +1,171 @@
-from prefect import task
-import sys
-from prefect.task_runners import ConcurrentTaskRunner, SequentialTaskRunner
-from prefect import runtime
-# from prefect_dask.task_runners import DaskTaskRunner
-from typing import Dict, List
-import yaml
-import importlib
-import os
-from kube_watch.models.workflow import WorkflowConfig, BatchFlowConfig, Task
-from kube_watch.enums.workflow import ParameterType, TaskRunners, TaskInputsType
-from kube_watch.modules.logic.merge import merge_logical_list
-
-def load_workflow_config(yaml_file) -> WorkflowConfig:
-    with open(yaml_file, 'r') as file:
-        data = yaml.safe_load(file)
-    return WorkflowConfig(**data['workflow'])
-
-
-def load_batch_config(yaml_file) -> BatchFlowConfig:
-    with open(yaml_file, 'r') as file:
-        data = yaml.safe_load(file)
-    return BatchFlowConfig(**data['batchFlows'])
-
-
-
-# def execute_task(func, *args, name="default_task_name", **kwargs):
-#     @task(name=name)
-#     def func_task():
-#         return func(*args, **kwargs)
-#     return func_task
-
-
-def func_task(name="default_task_name", task_input_type: TaskInputsType = TaskInputsType.ARG):
-    if task_input_type == TaskInputsType.ARG:
-        @task(name=name)
-        def execute_task(func, *args, **kwargs):
-            return func(*args, **kwargs)
-        return execute_task
-    if task_input_type == TaskInputsType.DICT:
-        @task(name=name)
-        def execute_task_dict(func, dict_inp):
-            return func(dict_inp)
-        return execute_task_dict
-    raise ValueError(f'Unknow Task Input Type. It should either be {TaskInputsType.ARG} or {TaskInputsType.DICT} but {task_input_type} is provided.')
-
-
-# @task
-# def execute_task(func, *args, **kwargs):
-#     return func(*args, **kwargs)
-
-
-
-def get_task_function(module_name, task_name, plugin_path=None):
-    """
-    Fetch a function directly from a specified module.
-    
-    Args:
-        module_name (str): The name of the module to import the function from. e.g. providers.aws
-        task_name (str): The name of the function to fetch from the module.
-        plugin_path (ster): define for external modules
-        
-    Returns:
-        function: The function object fetched from the module.
-    """
-    try:
-        if plugin_path:
-            # Temporarily prepend the plugin path to sys.path to find the module
-            module_path = os.path.join(plugin_path, *module_name.split('.')) + '.py'
-            module_spec = importlib.util.spec_from_file_location(module_name, module_path)
-            module = importlib.util.module_from_spec(module_spec)
-            module_spec.loader.exec_module(module)
-        else:
-            # Standard import from the internal library path
-            module = importlib.import_module(f"kube_watch.modules.{module_name}")
-        
-        return getattr(module, task_name)
-    except ImportError as e:
-        raise ImportError(f"Unable to import module '{module_name}': {e}")
-    except AttributeError as e:
-        raise AttributeError(f"The module '{module_name}' does not have a function named '{task_name}': {e}")
-    # finally:
-    #     if plugin_path:
-    #         # Remove the plugin path from sys.path after importing
-    #         sys.path.pop(0)  # Using pop(0) is safer in the context of insert(0, plugin_path)
-
-
-
-def resolve_parameter_value(param):
-    if param.type == ParameterType.FROM_ENV:
-        return os.getenv(param.value, '')  # Default to empty string if env var is not set
-    if param.type == ParameterType.FROM_FLOW:
-        return runtime.flow_run.parameters.get(param.value, '')
-    return param.value
-
-def prepare_task_inputs(parameters):
-    return {param.name: resolve_parameter_value(param) for param in parameters}
-
-
-def prepare_task_inputs_from_dep(task_data: Task, task_inputs: Dict, tasks):
-    for dep in task_data.dependency:
-        par_task   = tasks[dep.taskName]
-        par_res    = par_task.result()
-        if dep.inputParamName != None:
-            task_inputs.update({dep.inputParamName: par_res})
-
-    return task_inputs
-
-
-def resolve_conditional(task_data: Task, tasks):
-    lst_bools = []
-    for task_name in task_data.conditional.tasks:
-        if task_name not in tasks:
-            return False
-        
-        par_task   = tasks[task_name]
-        lst_bools.append(par_task.result())
-    return merge_logical_list(lst_bools, task_data.conditional.operation)
-    
-
-
-
-def submit_task(task_name, task_data, task_inputs, func):
-    execute_task = func_task(name=task_name, task_input_type=task_data.inputsArgType)
-    if task_data.inputsArgType == TaskInputsType.ARG:
-        return execute_task.submit(func, **task_inputs)
-    if task_data.inputsArgType == TaskInputsType.DICT:
-        return execute_task.submit(func, dict_inp=task_inputs)
-    raise ValueError("Unknown Input Arg Type.")
-
-
-
-def resolve_runner(runner):
-    if runner == TaskRunners.CONCURRENT:
-        return ConcurrentTaskRunner
-    if runner == TaskRunners.SEQUENTIAL:
-        return SequentialTaskRunner
-    if runner == TaskRunners.DASK:
-        raise ValueError("Dask Not Implemented")
-        # return DaskTaskRunner
-    if runner == TaskRunners.RAY:
-        raise ValueError("Ray Not Implemented")
-        # return RayTaskRunner
-    raise ValueError("Invalid task runner type")
-
-
-def filter_attributes(obj):
-    import uuid
-    from collections.abc import Iterable
-    import inspect
-
-    def is_simple(value):
-        """ Check if the value is a simple data type or a collection of simple data types """
-        if isinstance(value, (int, float, str, bool, type(None), uuid.UUID)):
-            return True
-        if isinstance(value, dict):
-            return all(is_simple(k) and is_simple(v) for k, v in value.items())
-        if isinstance(value, Iterable) and not isinstance(value, (str, bytes)):
-            return all(is_simple(item) for item in value)
-        return False
-
-    result = {}
-    for attr in dir(obj):
-        # Avoid magic methods and attributes
-        if attr.startswith("__") and attr.endswith("__"):
-            continue
-        value = getattr(obj, attr)
-        # Filter out methods and check if the attribute value is simple
-        if not callable(value) and not inspect.isclass(value) and is_simple(value):
-            result[attr] = value
+from prefect import task
+import sys
+from prefect.task_runners import ConcurrentTaskRunner, ThreadPoolTaskRunner
+from prefect import runtime
+# from prefect_dask.task_runners import DaskTaskRunner
+from typing import Dict, List
+import yaml
+import importlib
+import os
+from kube_watch.models.workflow import WorkflowConfig, BatchFlowConfig, Task
+from kube_watch.enums.workflow import ParameterType, TaskRunners, TaskInputsType
+from kube_watch.modules.logic.merge import merge_logical_list
+
+def load_workflow_config(yaml_file) -> WorkflowConfig:
+    with open(yaml_file, 'r') as file:
+        data = yaml.safe_load(file)
+    return WorkflowConfig(**data['workflow'])
+
+
+def load_batch_config(yaml_file) -> BatchFlowConfig:
+    with open(yaml_file, 'r') as file:
+        data = yaml.safe_load(file)
+    return BatchFlowConfig(**data['batchFlows'])
+
+
+
+# def execute_task(func, *args, name="default_task_name", **kwargs):
+#     @task(name=name)
+#     def func_task():
+#         return func(*args, **kwargs)
+#     return func_task
+
+
+def func_task(name="default_task_name", task_input_type: TaskInputsType = TaskInputsType.ARG):
+    if task_input_type == TaskInputsType.ARG:
+        @task(name=name)
+        def execute_task(func, *args, **kwargs):
+            return func(*args, **kwargs)
+        return execute_task
+    if task_input_type == TaskInputsType.DICT:
+        @task(name=name)
+        def execute_task_dict(func, dict_inp):
+            return func(dict_inp)
+        return execute_task_dict
+    raise ValueError(f'Unknow Task Input Type. It should either be {TaskInputsType.ARG} or {TaskInputsType.DICT} but {task_input_type} is provided.')
+
+
+# @task
+# def execute_task(func, *args, **kwargs):
+#     return func(*args, **kwargs)
+
+
+
+def get_task_function(module_name, task_name, plugin_path=None):
+    """
+    Fetch a function directly from a specified module.
+    
+    Args:
+        module_name (str): The name of the module to import the function from. e.g. providers.aws
+        task_name (str): The name of the function to fetch from the module.
+        plugin_path (ster): define for external modules
+        
+    Returns:
+        function: The function object fetched from the module.
+    """
+    try:
+        if plugin_path:
+            # Temporarily prepend the plugin path to sys.path to find the module
+            module_path = os.path.join(plugin_path, *module_name.split('.')) + '.py'
+            module_spec = importlib.util.spec_from_file_location(module_name, module_path)
+            module = importlib.util.module_from_spec(module_spec)
+            module_spec.loader.exec_module(module)
+        else:
+            # Standard import from the internal library path
+            module = importlib.import_module(f"kube_watch.modules.{module_name}")
+        
+        return getattr(module, task_name)
+    except ImportError as e:
+        raise ImportError(f"Unable to import module '{module_name}': {e}")
+    except AttributeError as e:
+        raise AttributeError(f"The module '{module_name}' does not have a function named '{task_name}': {e}")
+    # finally:
+    #     if plugin_path:
+    #         # Remove the plugin path from sys.path after importing
+    #         sys.path.pop(0)  # Using pop(0) is safer in the context of insert(0, plugin_path)
+
+
+
+def resolve_parameter_value(param):
+    if param.type == ParameterType.FROM_ENV:
+        return os.getenv(param.value, '')  # Default to empty string if env var is not set
+    if param.type == ParameterType.FROM_FLOW:
+        return runtime.flow_run.parameters.get(param.value, '')
+    return param.value
+
+def prepare_task_inputs(parameters):
+    return {param.name: resolve_parameter_value(param) for param in parameters}
+
+
+def prepare_task_inputs_from_dep(task_data: Task, task_inputs: Dict, tasks):
+    for dep in task_data.dependency:
+        par_task   = tasks[dep.taskName]
+        par_res    = par_task.result()
+        if dep.inputParamName != None:
+            task_inputs.update({dep.inputParamName: par_res})
+
+    return task_inputs
+
+
+def resolve_conditional(task_data: Task, tasks):
+    lst_bools = []
+    for task_name in task_data.conditional.tasks:
+        if task_name not in tasks:
+            return False
+        
+        par_task   = tasks[task_name]
+        lst_bools.append(par_task.result())
+    return merge_logical_list(lst_bools, task_data.conditional.operation)
+    
+
+
+
+def submit_task(task_name, task_data, task_inputs, func):
+    execute_task = func_task(name=task_name, task_input_type=task_data.inputsArgType)
+    if task_data.inputsArgType == TaskInputsType.ARG:
+        return execute_task.submit(func, **task_inputs)
+    if task_data.inputsArgType == TaskInputsType.DICT:
+        return execute_task.submit(func, dict_inp=task_inputs)
+    raise ValueError("Unknown Input Arg Type.")
+
+
+
+def resolve_runner(runner):
+    if runner == TaskRunners.CONCURRENT:
+        return ConcurrentTaskRunner
+    if runner == TaskRunners.SEQUENTIAL:
+        return ThreadPoolTaskRunner(max_workers=1)
+    if runner == TaskRunners.DASK:
+        raise ValueError("Dask Not Implemented")
+        # return DaskTaskRunner
+    if runner == TaskRunners.RAY:
+        raise ValueError("Ray Not Implemented")
+        # return RayTaskRunner
+    raise ValueError("Invalid task runner type")
+
+
+def filter_attributes(obj):
+    import uuid
+    from collections.abc import Iterable
+    import inspect
+
+    def is_simple(value):
+        """ Check if the value is a simple data type or a collection of simple data types """
+        if isinstance(value, (int, float, str, bool, type(None), uuid.UUID)):
+            return True
+        if isinstance(value, dict):
+            return all(is_simple(k) and is_simple(v) for k, v in value.items())
+        if isinstance(value, Iterable) and not isinstance(value, (str, bytes)):
+            return all(is_simple(item) for item in value)
+        return False
+
+    result = {}
+    for attr in dir(obj):
+        # Avoid magic methods and attributes
+        if attr.startswith("__") and attr.endswith("__"):
+            continue
+        value = getattr(obj, attr)
+        # Filter out methods and check if the attribute value is simple
+        if not callable(value) and not inspect.isclass(value) and is_simple(value):
+            result[attr] = value
     return result