ctm-python-client 2.1.2__py3-none-any.whl

ctm_python_client/core/workflow.py

@@ -0,0 +1,413 @@
+import abc
+import typing
+import attrs
+import json
+import pathlib
+import random
+import tempfile
+import collections
+import copy
+
+from aapi import *
+from ctm_python_client.core.comm import AAPIClientResponse, AbstractAAPIClient, Environment, EnvironmentMode, OnPremAAPIClient, SaasAAPIClient, sanitize_output
+from ctm_python_client.core.monitoring import RunMonitor
+
+__all__ = ['AbstractWorkflow', 'WorkflowDefaults', 'BaseWorkflow', 'Workflow']
+
+
+class AbstractWorkflow(abc.ABC):
+    @abc.abstractclassmethod
+    def add(self, ):
+        pass
+
+    @abc.abstractclassmethod
+    def chain(self, ):
+        pass
+
+    @abc.abstractclassmethod
+    def connect(self,):
+        pass
+
+    @abc.abstractclassmethod
+    def get(self,):
+        pass
+
+    @abc.abstractclassmethod
+    def get_objects(self,):
+        pass
+
+    @abc.abstractclassmethod
+    def dump_json(self,):
+        pass
+
+    @abc.abstractclassmethod
+    def dumps_json(self,):
+        pass
+
+
+# class AbstractWorkflowManager(AbstractWorkflow):
+
+#     @abc.abstractclassmethod
+#     def build(self, ):
+#         pass
+
+#     @abc.abstractclassmethod
+#     def deploy(self,):
+#         pass
+
+#     @abc.abstractclassmethod
+#     def run(self,):
+#         pass
+
+
+@attrs.define(kw_only=True)
+class WorkflowDefaults:
+    controlm_server: str = attrs.field(
+        default=None, metadata={'applyon': ['Folder', 'SimpleFolder']})
+    run_as: str = attrs.field(default=None, metadata={'applyon': [
+                              'Folder', 'SimpleFolder', 'SubFolder', 'Job']})
+    host: str = attrs.field(default=None, metadata={'applyon': ['Job']})
+    when: Job.When = attrs.field(default=None, metadata={
+                                 'applyon': ['Folder', 'Job']})
+    application: str = attrs.field(default=None, metadata={'applyon': [
+                                   'Folder', 'SimpleFolder', 'SubFolder', 'Job']})
+    sub_application: str = attrs.field(default=None, metadata={'applyon': [
+                                       'Folder', 'SimpleFolder', 'SubFolder', 'Job']})
+
+
+class BaseWorkflow(AbstractWorkflow):
+    def __init__(self, defaults: WorkflowDefaults = None) -> None:
+        super().__init__()
+
+        self._definitions = {}
+        self._connections = {}
+
+        self.defaults = defaults if defaults else WorkflowDefaults()
+
+    def clear_all(self):
+        self._definitions = {}
+        self._connections = {}
+
+    def _apply_defaults_for_type(self, obj: typing.Any, type_: str):
+        for field in attrs.fields(WorkflowDefaults):
+            if type_ in field.metadata.get('applyon') \
+                    and self.defaults.__getattribute__(field.name):
+                if obj.__getattribute__(field.name) is None:
+                    obj.__setattr__(
+                        field.name, self.defaults.__getattribute__(field.name))
+
+    def _apply_defaults_for_job(self, obj: Job):
+        self._apply_defaults_for_type(obj, 'Job')
+
+    def _apply_defaults_for_folder(self, obj: Folder):
+        self._apply_defaults_for_type(obj, 'Folder')
+
+    def _apply_defaults_for_SimpleFolder(self, obj: SimpleFolder):
+        self._apply_defaults_for_type(obj, 'SimpleFolder')
+
+    def _apply_defaults_for_SubFolder(self, obj: SubFolder):
+        self._apply_defaults_for_type(obj, 'SubFolder')
+
+    def add(self, obj: AAPIObject, inpath: str = None, allow_creation: bool = True):
+        # apply defaults:
+        if isinstance(obj, Job):
+            self._apply_defaults_for_job(obj)
+        elif isinstance(obj, SubFolder):
+            self._apply_defaults_for_SubFolder(obj)
+        elif isinstance(obj, Folder):
+            self._apply_defaults_for_folder(obj)
+        elif isinstance(obj, SimpleFolder):
+            self._apply_defaults_for_SimpleFolder(obj)
+        else:
+            pass
+
+        if inpath:
+            if isinstance(obj, SubFolder) or isinstance(obj, Job):
+                try:
+                    folder = self.get(inpath)
+                except Exception as e:
+                    if allow_creation:
+                        folder = None
+                    else:
+                        raise e
+                if folder:
+                    if isinstance(obj, Job):
+                        folder.job_list.append(obj)
+                    else:
+                        folder.sub_folder_list.append(obj)
+                else:
+                    # Folder does not exist, create folder
+                    folder, parent_folder = self.create_folder_hierarchy(inpath)
+                    self._apply_defaults_for_folder(folder)
+                    if isinstance(obj, Job):
+                        parent_folder.job_list.append(obj)
+                    else:
+                        parent_folder.sub_folder_list.append(obj)
+
+                    self._definitions[folder.object_name] = folder
+            else:
+                raise Exception(
+                    f'Cannot add object of type {type(obj)} into Folder. Only SubFolder and Job is allowed')
+        else:
+            if not (isinstance(obj, SimpleFolder)
+                    or isinstance(obj, Folder)
+                    or isinstance(obj, ConnectionProfile)
+                    ):
+                raise Exception(
+                    f'Cannot add object of type {type(obj)} as root definition. Only Folder and SimpleFolder is allowed')
+            else:
+                self._definitions[obj.object_name] = obj
+
+        if inpath:
+            return '/'.join([inpath, obj.object_name])
+        else:
+            return obj.object_name
+
+    def connect(self, srcpath: str, destpath: str, *, inpath: str = None):
+        if inpath:
+            srcpath = f'{inpath}/{srcpath}'
+            destpath = f'{inpath}/{destpath}'
+
+        obj_src = self.get(srcpath)
+        obj_dest = self.get(destpath)
+
+        if not isinstance(obj_src, AAPIJob) or not isinstance(obj_dest, AAPIJob):
+            raise Exception(
+                f'Cannot connect objects of type {type(obj_src)} with {type(obj_dest)}. Connections can be made from/to Folder, SubFolder and Job')
+
+        key_name = (srcpath, destpath)
+        event_name = f'{obj_src.object_name.replace(" ", "_")}-TO-{obj_dest.object_name.replace(" ", "_")}'
+        if event_name in self._connections:
+            self._connections[event_name].append(key_name)
+            count = len(self._connections[event_name])+1
+            event_name = f'{event_name}-{count}'
+        else:
+            # self._connections[key_name] = {'count': 1, 'name': event_name}
+            self._connections[event_name] = [key_name]
+
+        # Write the add, wait and delete events
+
+        if not obj_src.add_events_list:
+            obj_src.add_events_list.append(
+                AddEvents('events_to_add', events=[]))
+        obj_src.add_events_list[-1].events.append(
+            ConditionOutAdd(event=event_name))
+
+        if not obj_dest.wait_for_events_list:
+            obj_dest.wait_for_events_list.append(
+                WaitForEvents('events_to_wait', events=[]))
+        obj_dest.wait_for_events_list[-1].events.append(
+            ConditionIn(event=event_name))
+
+        if not obj_dest.delete_events_list:
+            obj_dest.delete_events_list.append(
+                DeleteEvents('events_to_delete', events=[]))
+        obj_dest.delete_events_list[-1].events.append(
+            ConditionOutDelete(event=event_name))
+
+    def chain(self, obj_list: typing.List[AAPIJob], inpath: str):
+        objpaths = []
+        objname = [o.object_name for o in obj_list]
+        clct = collections.Counter(objname)
+        dupl = []
+        for c in clct:
+            if clct[c] > 1:
+                dupl.append(f'"{c}"')
+
+        if dupl:
+            raise Exception(f'Multiple definitions for: {" , ".join(dupl)}')
+
+        objs_paths = []
+        for obj in obj_list:
+            objs_paths.append(self.add(obj, inpath=inpath))
+            objpaths.append(obj.object_name)
+
+        connections = [(objpaths[i], objpaths[i+1])
+                       for i, _ in enumerate(objpaths[:-1])]
+        for conn in connections:
+            self.connect(conn[0], conn[1], inpath=inpath)
+
+        return objs_paths
+
+    def get_objects(self, path: str) -> typing.List[AAPIObject]:
+        pathparts = path.split('/')
+        name = pathparts[-1]
+        pathparts = pathparts[:-1]
+
+        current_object = self._definitions
+
+        for i, p in enumerate(pathparts):
+            if isinstance(current_object, dict):
+                try:
+                    current_object = current_object[p]
+                except KeyError as e:
+                    raise Exception(f'Path {pathparts[:i]} is invalid')
+
+            elif isinstance(current_object, Folder) or isinstance(current_object, SubFolder):
+                try:
+                    current_object = [
+                        o for o in current_object.sub_folder_list if o.object_name == p][0]
+                except IndexError as e:
+                    raise Exception(f'Path {pathparts[:i]} is invalid')
+            else:
+                raise Exception(f'Path {pathparts[:i]} is invalid')
+
+        if isinstance(current_object, dict):
+            try:
+                res = [current_object[name]]
+                return res
+            except KeyError as e:
+                raise Exception(
+                    f'Cannot find object {name} in path: {"/".join(pathparts)}')
+
+        elif isinstance(current_object, Folder) or isinstance(current_object, SubFolder):
+            res = [o for o in current_object.sub_folder_list +
+                   current_object.job_list if o.object_name == name]
+            if not res:
+                raise Exception(
+                    f'Cannot find object {name} in path: {"/".join(pathparts)}')
+            else:
+                return res
+        elif isinstance(current_object, SimpleFolder):
+            res = [o for o in current_object.job_list if o.object_name == name]
+            if not res:
+                raise Exception(
+                    f'Cannot find object {name} in path: {"/".join(pathparts)}')
+            else:
+                return res
+        else:
+            raise Exception(
+                f'Cannot find object {name} in path: {"/".join(pathparts)}')
+
+    def get(self, path: str, default=None) -> AAPIObject:
+        if default is None:
+            return self.get_objects(path)[0]
+        else:
+            try:
+                return self.get_objects(path)[0]
+            except:
+                return default
+
+
+
+    def create_folder_hierarchy(self, path : str):
+        parts = path.split('/')
+        folder = self.get(parts[0], default = Folder(parts[0]))
+        parent = folder
+        for i,o in enumerate(parts[1:]):
+            try:
+                subfolder = [sf for sf in parent.sub_folder_list if sf.object_name == o][0]
+            except IndexError:
+                subfolder = SubFolder(o)
+                parent.sub_folder_list.append(subfolder)
+            parent = subfolder
+
+        return folder,parent
+
+
+    def dumps_json(self, indent=None):
+        return json.dumps({k: v.as_aapi_dict() for (k, v) in self._definitions.items()}, indent=indent)
+
+    def dump_json(self, f, indent=None):
+        return json.dump({k: v.as_aapi_dict() for (k, v) in self._definitions.items()}, f, indent=indent)
+
+    def copy_from(self, workflow):
+        self._definitions = copy.deepcopy(workflow._definitions)
+        self._connections = copy.deepcopy(workflow._connections)
+
+
+
+class Workflow(BaseWorkflow):
+
+    @staticmethod
+    def workbench(host : str = 'localhost', port : str = '8443'):
+        return Workflow(Environment.create_workbench(host, port), WorkflowDefaults(run_as='workbench'))
+
+    def __init__(self, environment: Environment, defaults: WorkflowDefaults = None, skip_initial_authentication: bool = False) -> None:
+        super().__init__(defaults=defaults)
+
+        self.environment = environment
+
+        self.aapiclient: AbstractAAPIClient = None
+        if self.environment.mode == EnvironmentMode.SAAS:
+            self.aapiclient = SaasAAPIClient(
+                environment.endpoint, environment.credentials)
+        else:
+            self.aapiclient = OnPremAAPIClient(
+                environment.endpoint, environment.credentials)
+
+        if not skip_initial_authentication:
+            self.authenticate()
+
+    def authenticate(self):
+        self.aapiclient.authenticate()
+
+    @sanitize_output
+    def build(self, skip_login: bool = False, file_path: str = None, delete_afterwards: bool = True) -> AAPIClientResponse:
+        if not skip_login:
+            self.aapiclient.authenticate()
+
+        if not file_path:
+            file_path = f'{tempfile.gettempdir()}/temp_{random.randint(1000,9999)}.json'
+
+        fpath = pathlib.Path(file_path)
+
+        with open(fpath.resolve(), 'w') as f:
+            self.dump_json(f)
+
+        try:
+            res = self.aapiclient.build_api.build_file(fpath.resolve())
+            return res
+        except Exception as e:
+            raise e
+        finally:
+            if delete_afterwards:
+                fpath.unlink()
+
+    @sanitize_output
+    def deploy(self, skip_login: bool = False, file_path: str = None, delete_afterwards: bool = True) -> AAPIClientResponse:
+        if not skip_login:
+            self.aapiclient.authenticate()
+
+        if not file_path:
+            file_path = f'{tempfile.gettempdir()}/temp_{random.randint(1000,9999)}.json'
+
+        fpath = pathlib.Path(file_path)
+
+        with open(fpath.resolve(), 'w') as f:
+            self.dump_json(f)
+
+        try:
+            res = self.aapiclient.deploy_api.deploy_file(fpath.resolve())
+            return res
+        except Exception as e:
+            raise e
+        finally:
+            if delete_afterwards:
+                fpath.unlink()
+
+    # @sanitize_output
+    def run(self, skip_login: bool = False, file_path: str = None, delete_afterwards: bool = True, open_in_browser=False) -> RunMonitor:
+        if not skip_login:
+            self.aapiclient.authenticate()
+
+        if not file_path:
+            file_path = f'{tempfile.gettempdir()}/temp_{random.randint(1000,9999)}.json'
+
+        fpath = pathlib.Path(file_path)
+
+        with open(fpath.resolve(), 'w') as f:
+            self.dump_json(f)
+
+        try:
+            res = self.aapiclient.run_api.run_jobs(fpath.resolve())
+            # return res
+            run_ =  RunMonitor(res.run_id, self.aapiclient, monitor_page_uri= res.monitor_page_uri)
+            if open_in_browser:                
+                run_.open_in_browser()
+            return run_
+        except Exception as e:
+            raise e
+        finally:
+            if delete_afterwards:
+                fpath.unlink()

ctm_python_client/ext/autogen.py

@@ -0,0 +1,243 @@
+import jinja2
+import urllib3
+import typing
+import requests
+import attrs
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+__all__ = ['generate_ai_plugins_classes']
+
+IF_PLUGINS = [
+    'ABA092022',
+    'GBA032023',
+    'GDF032022',
+    'GBQ102022',
+    'AEC082022',
+    'TDM052022',
+    'GDP042022',
+    'ZSY062022',
+    'AQS012023',
+    'AHD062022',
+    'MFL022023',
+    'COM032023',
+    'ZML022023',
+    'GVM062022',
+    'ADF062021',
+    'ASF012023',
+    'ASM0220223',
+    'ICS032022',
+    'BOO032022',
+    'ZDX112021',
+    'ACS042023',
+    'AAR072022',
+    'UIP072021',
+    'ZLA112022',
+    'ATR122022',
+    'QLC092022',
+    'GLU062021',
+    'GDR052023',
+    'ADB112022',
+    'MBI042022',
+    'ABK042023',
+    'SNF092022',
+    'ADP122022',
+    'ZBA042022',
+    'DBX032022',
+    'ZFN032022',
+    'AEM072022',
+    'MFW022023',
+    'ZVM062022',
+    'DBT042023',
+    'AAT052023'
+]
+
+
+imports_str = '''from aapi import *
+import attrs
+
+'''
+template_job = jinja2.Template('''
+@attrs.define
+class {{plugin.class_name}}(AIJob):
+    _type = AIJob.type_field('{{plugin.type_field}}')
+
+    {% for p in plugin.mandatory_properties %}
+    {{p[0]}} = AIJob.field('{{p[1]}}')
+    {%endfor%}
+
+    {% for p in plugin.optional_properties %}
+    {{p[0]}} = AIJob.field_optional('{{p[1]}}', default='{{p[2]}}')
+    {%endfor%}    
+    ''')
+
+template_cp = jinja2.Template('''
+@attrs.define
+class {{plugin.class_name}}(AIConnectionProfile):
+    _type = AIConnectionProfile.type_field('{{plugin.type_field}}')
+
+    {% for p in plugin.mandatory_properties %}
+    {{p[0]}} = AIConnectionProfile.field('{{p[1]}}')
+    {%endfor%}
+
+    {% for p in plugin.optional_properties %}
+    {{p[0]}} = AIConnectionProfile.field_optional('{{p[1]}}', default='{{p[2]}}')
+    {%endfor%}    
+    ''')
+
+
+@attrs.define
+class Plugin:
+    job_name: str
+    cp_name: str
+    job_definition: typing.Dict[str, str]
+    cp_definition: typing.Dict[str, str]
+
+    plugin_type: str
+
+
+@attrs.define
+class AIPluginCustomDefinition:
+    class_name: str
+    type_field: str
+
+    mandatory_properties: typing.List[typing.Tuple[str]]
+    optional_properties: typing.List[typing.Tuple[str]]
+    comments: typing.List[str]
+
+
+def create_ai_custom_definitions(ai_plugin: Plugin) -> typing.Tuple[AIPluginCustomDefinition, AIPluginCustomDefinition]:
+    plugin_name = ai_plugin.job_name.split(':')[-1]
+    plugin_cp_name = ai_plugin.cp_name.split(':')[-1]
+    job = {
+        'class_name': 'AIJob'+plugin_name.capitalize().replace(" ", ""),
+        'type_field': plugin_name,
+        'mandatory_properties': [],
+        'optional_properties': [],
+
+        'comments': [f'Auto generated class for {plugin_name}']
+    }
+    cp = {
+        'class_name': 'ConnectionProfileAI'+plugin_cp_name.capitalize().replace(" ", ""),
+        'type_field': plugin_cp_name,
+        'mandatory_properties': [],
+        'optional_properties': [],
+        'comments': [f'Auto generated class for {plugin_cp_name}']
+    }
+
+    for k, v in ai_plugin.job_definition.items():
+        if not k.startswith('AI-'):
+            continue
+
+        if k == 'AI-Connection Profile':
+            continue
+
+        domain_name = v.get('DomainName')
+        if domain_name in ai_plugin.job_definition['Defaults']:
+            default = ai_plugin.job_definition['Defaults'][domain_name]
+
+            job['optional_properties'].append((
+                domain_name.split(r'%%UCM-')[1].lower(),
+                k.split('AI-')[1],
+                default
+            ))
+        else:
+            job['mandatory_properties'].append((
+                domain_name.split(r'%%UCM-')[1].lower(),
+                k.split('AI-')[1],
+                None
+            ))
+
+    for k, v in ai_plugin.cp_definition.items():
+        if not k.startswith('AI-'):
+            continue
+
+        if k in ['AI-Run As', 'AI-RunAs-Pass']:
+            continue
+
+        domain_name = v.get('DomainName')
+        if domain_name in ai_plugin.cp_definition['Defaults']:
+            default = ai_plugin.cp_definition['Defaults'][domain_name]
+
+            cp['optional_properties'].append((
+                domain_name.lower(),
+                k.split('AI-')[1],
+                default
+            ))
+        else:
+            cp['mandatory_properties'].append((
+                domain_name.lower(),
+                k.split('AI-')[1],
+                None
+            ))
+
+    return (AIPluginCustomDefinition(**job), AIPluginCustomDefinition(**cp))
+
+
+def get_all_plugins_versions_from_url(url: str) -> typing.Dict[str, str]:
+    r = requests.get(url=url, verify=False)
+
+    return r.json()
+
+
+def parse_plugin(url):
+    r = requests.get(url, verify=False)
+    plugin = {}
+
+    for k, v in r.json().items():
+        if k.startswith('Job:'):
+            plugin["job_name"] = k
+            plugin["job_definition"] = v
+            continue
+
+        if k.startswith('ConnectionProfile:'):
+            plugin["cp_name"] = k
+            plugin["cp_definition"] = v
+            continue
+
+    if 'Aliases' in r.json():
+        plugin['plugin_type'] = 'AI'
+    else:
+        plugin['plugin_type'] = 'IF'
+
+    return Plugin(**plugin)
+    # return plugin
+
+
+def get_all_plugins(host: str, port: str = '8443', exlude_list=[]):
+    versions = get_all_plugins_versions_from_url(
+        f'https://{host}:{port}/automation-api/ai-repo/deployed_plugins.json')
+
+    res = []
+    for plugin, version in versions.items():
+        if plugin in exlude_list:
+            continue
+
+        url = f'https://{host}:{port}/automation-api/ai-repo/{plugin}/v{version}/descriptorDictionary{plugin}.json'
+
+        res.append(parse_plugin(url))
+
+    return res
+
+
+def create_class_code(plugin_definitions):
+    job_str = template_job.render(plugin=plugin_definitions[0])
+    cp_str = template_cp.render(plugin=plugin_definitions[1])
+
+    return (job_str, cp_str)
+
+
+def generate_ai_plugins_classes(host: str = 'localhost', port: str = '8443', output_file: str = 'generated.py'):
+    plugins = get_all_plugins(host=host, port=port, exlude_list=IF_PLUGINS)
+    definitions = [create_ai_custom_definitions(plugin) for plugin in plugins]
+    with open(output_file, 'w') as f:
+        f.write(imports_str)
+
+        for definition in definitions:
+            def_str = create_class_code(definition)
+            f.write(f'''
+
+# Auto generated code for AI Plugin {definition[0].type_field}
+            ''')
+            f.write(def_str[0])  # JOB
+            f.write(def_str[1])  # Connection Profile

ctm_python_client/ext/credential.py

@@ -0,0 +1,18 @@
+from ctm_python_client.core.credential import AbstractCredentials
+
+import keyring
+
+__all__ = ['KeyringCredentials']
+
+class KeyringCredentials(AbstractCredentials):
+    def __init__(self, service: str, username: str) -> None:
+        super().__init__()
+
+        self.service = service
+        self.username = username
+
+    def get_username(self):
+        return self.username
+
+    def get_password(self):
+        return keyring.get_password(self.service, self.username)