illumio-pylo 0.2.5__py3-none-any.whl

illumio_pylo/Organization.py

@@ -0,0 +1,258 @@
+from typing import Optional, List, Callable
+import json
+import os
+import getpass
+import illumio_pylo as pylo
+from .API.JsonPayloadTypes import PCEObjectsJsonStructure, PCECacheFileJsonStructure
+from .API.CredentialsManager import get_credentials_from_file
+
+
+class Organization:
+
+    def __init__(self, org_id):
+        self.id: int = org_id
+        self.connector: Optional['pylo.APIConnector'] = None
+        self.LabelStore: 'pylo.LabelStore' = pylo.LabelStore(self)
+        self.IPListStore: 'pylo.IPListStore' = pylo.IPListStore(self)
+        self.WorkloadStore: 'pylo.WorkloadStore' = pylo.WorkloadStore(self)
+        self.VirtualServiceStore: 'pylo.VirtualServiceStore' = pylo.VirtualServiceStore(self)
+        self.AgentStore: 'pylo.AgentStore' = pylo.AgentStore(self)
+        self.ServiceStore: 'pylo.ServiceStore' = pylo.ServiceStore(self)
+        self.RulesetStore: 'pylo.RulesetStore' = pylo.RulesetStore(self)
+        self.SecurityPrincipalStore: 'pylo.SecurityPrincipalStore' = pylo.SecurityPrincipalStore(self)
+        self.pce_version: Optional['pylo.SoftwareVersion'] = None
+
+    def load_from_cached_file(self, fqdn: str, no_exception_if_file_does_not_exist=False) -> bool:
+        # filename should be like 'cache_xxx.yyy.zzz.json'
+        filename = 'cache_' + fqdn + '.json'
+
+        if os.path.isfile(filename):
+            # now we try to open that JSON file
+            with open(filename) as json_file:
+                data: PCECacheFileJsonStructure = json.load(json_file)
+                if 'pce_version' not in data:
+                    raise pylo.PyloEx("Cannot find PCE version in cache file")
+                self.pce_version = pylo.SoftwareVersion(data['pce_version'])
+                if 'data' not in data:
+                    raise pylo.PyloEx("Cache file '%s' was found and successfully loaded but no 'data' object could be found" % filename)
+                self.load_from_json(data['data'])
+                return True
+
+        if no_exception_if_file_does_not_exist:
+            return False
+
+        raise pylo.PyloEx("Cache file '%s' was not found!" % filename)
+
+    @staticmethod
+    def get_from_cache_file(fqdn: str) -> 'pylo.Organization':
+        org = pylo.Organization(1)
+        org.load_from_cached_file(fqdn)
+        return org
+
+    @staticmethod
+    def get_from_api_using_credential_file(fqdn_or_profile_name: str = None,
+                                           credential_file: str = None,
+                                           list_of_objects_to_load: Optional[List['pylo.ObjectTypes']] = None,
+                                           include_deleted_workloads: bool = False,
+                                           callback_api_objects_downloaded: Callable = None) -> 'Organization':
+        """
+        Credentials files will be looked for in the following order:
+        1. The path provided in the credential_file argument
+        2. The path provided in the Pylo_CREDENTIAL_FILE environment variable
+        3. The path ~/.pylo/credentials.json
+        4. Current working directory credentials.json
+        :param fqdn_or_profile_name:
+        :param credential_file:
+        :param list_of_objects_to_load:
+        :param include_deleted_workloads:
+        :param callback_api_objects_downloaded: callback function that will be called after each API has finished downloading all objects
+        :return:
+        """
+        credentials = get_credentials_from_file(fqdn_or_profile_name, credential_file)
+
+        connector = pylo.APIConnector(fqdn=credentials.fqdn, port=credentials.port,
+                                      apiuser=credentials.api_user, apikey=credentials.api_key,
+                                      org_id=credentials.org_id,
+                                      skip_ssl_cert_check=not credentials.verify_ssl, name=fqdn_or_profile_name)
+
+        objects = connector.get_pce_objects(list_of_objects_to_load=list_of_objects_to_load,
+                                            include_deleted_workloads=include_deleted_workloads)
+
+        if callback_api_objects_downloaded is not None:
+            callback_api_objects_downloaded()
+
+        org = Organization(1)
+        org.load_from_json(objects,list_of_objects_to_load=list_of_objects_to_load)
+
+        return org
+
+    def load_from_cache_or_saved_credentials(self, fqdn: str, include_deleted_workloads=False, prompt_for_api_key_if_missing=True):
+        """
+        Load the organization from a cache file on disk or default to the API
+        :param fqdn: the hostname of the PCE
+        :param include_deleted_workloads: if True, deleted workloads will be loaded from the API
+        :param prompt_for_api_key_if_missing: if True, the user will be prompted for an API key if it's unknown
+        :return:
+        """
+        if not self.load_from_cached_file(fqdn, no_exception_if_file_does_not_exist=True):
+            self.load_from_saved_credentials(fqdn, include_deleted_workloads=include_deleted_workloads, prompt_for_api_key=prompt_for_api_key_if_missing)
+
+    def load_from_saved_credentials(self, fqdn: str, include_deleted_workloads=False, prompt_for_api_key=False,
+                                    list_of_objects_to_load: Optional[List[str]] = None):
+        separator_pos = fqdn.find(':')
+        port = 8443
+
+        if separator_pos > 0:
+            port = fqdn[separator_pos + 1:]
+            fqdn = fqdn[0:separator_pos]
+
+        connector = pylo.APIConnector.create_from_credentials_in_file(fqdn)
+        if connector is None:
+            if not prompt_for_api_key:
+                raise pylo.PyloEx('Cannot find credentials for host {}'.format(fqdn))
+            print('Cannot find credentials for host "{}".\nPlease input an API user:'.format(fqdn), end='')
+            user = input()
+            password = getpass.getpass()
+            connector = pylo.APIConnector(fqdn, port, user, password, skip_ssl_cert_check=True, org_id=self.id)
+
+        self.load_from_api(connector, include_deleted_workloads=include_deleted_workloads,
+                           list_of_objects_to_load=list_of_objects_to_load)
+
+    def load_from_json(self, data: PCEObjectsJsonStructure,
+                       list_of_objects_to_load: Optional[List['pylo.ObjectTypes']] = None)\
+            -> None:
+        """
+        Load the organization from a JSON structure, mostly for developers use only
+        """
+        object_to_load = {}
+        if list_of_objects_to_load is not None:
+            all_types = pylo.APIConnector.get_all_object_types()
+            for object_type in list_of_objects_to_load:
+                if object_type not in all_types:
+                    raise pylo.PyloEx("Unknown object type '{}'".format(object_type))
+                object_to_load[object_type] = True
+        else:
+            object_to_load = pylo.APIConnector.get_all_object_types()
+
+        if self.pce_version is None:
+            raise pylo.PyloEx('Organization has no "version" specified')
+
+        self.LabelStore.load_label_dimensions(data.get('label_dimensions'))
+
+        if 'labels' in object_to_load:
+            if 'labels' not in data:
+                raise Exception("'labels' was not found in json data")
+            self.LabelStore.load_labels_from_json(data['labels'])
+
+        if 'labelgroups' in object_to_load:
+            if 'labelgroups' not in data:
+                raise Exception("'labelgroups' was not found in json data")
+            self.LabelStore.load_label_groups_from_json(data['labelgroups'])
+
+        if 'iplists' in object_to_load:
+            if 'iplists' not in data:
+                raise Exception("'iplists' was not found in json data")
+            self.IPListStore.load_iplists_from_json(data['iplists'])
+
+        if 'services' in object_to_load:
+            if 'services' not in data:
+                raise Exception("'services' was not found in json data")
+            self.ServiceStore.load_services_from_json(data['services'])
+
+        if 'workloads' in object_to_load:
+            if 'workloads' not in data:
+                raise Exception("'workloads' was not found in json data")
+            self.WorkloadStore.load_workloads_from_json(data['workloads'])
+
+        if 'virtual_services' in object_to_load:
+            if 'virtual_services' not in data:
+                raise Exception("'virtual_services' was not found in json data")
+            self.VirtualServiceStore.load_virtualservices_from_json(data['virtual_services'])
+
+        if 'security_principals' in object_to_load:
+            if 'security_principals' not in data:
+                raise Exception("'security_principals' was not found in json data")
+            self.SecurityPrincipalStore.load_principals_from_json(data['security_principals'])
+
+        if 'rulesets' in object_to_load:
+            if 'rulesets' not in data:
+                raise Exception("'rulesets' was not found in json data")
+            self.RulesetStore.load_rulesets_from_json(data['rulesets'])
+
+    def load_from_api(self, con: pylo.APIConnector, include_deleted_workloads=False,
+                      list_of_objects_to_load: Optional[List['pylo.ObjectTypes']] = None):
+        """
+        Load the organization from the API with the API Connector provided. Mostly intended for developers use only
+        :param con:
+        :param include_deleted_workloads:
+        :param list_of_objects_to_load:
+        :return:
+        """
+        self.pce_version = con.get_software_version()
+        self.connector = con
+        return self.load_from_json(self.get_config_from_api(con, include_deleted_workloads=include_deleted_workloads,
+                                                            list_of_objects_to_load=list_of_objects_to_load))
+
+    @staticmethod
+    def create_fake_empty_config() -> PCEObjectsJsonStructure:
+        """
+        Create a fake empty config, mostly for developers use only
+        :return:
+        """
+        data = {}
+        for object_type in pylo.APIConnector.get_all_object_types().values():
+            data[object_type] = []
+        return data
+
+    def get_config_from_api(self, con: pylo.APIConnector, include_deleted_workloads=False,
+                            list_of_objects_to_load: Optional[List[str]] = None) -> PCEObjectsJsonStructure:
+        """
+        Get the config/objects from the API using the API connector provided
+        :param con:
+        :param include_deleted_workloads:
+        :param list_of_objects_to_load:
+        :return:
+        """
+        self.connector = con
+        return con.get_pce_objects(include_deleted_workloads=include_deleted_workloads,
+                                   list_of_objects_to_load=list_of_objects_to_load)
+
+    def stats_to_str(self, padding='') -> str:
+        """ Dumps basic stats about the organization
+        :param padding: String to be added at the beginning of each line
+        Example:
+        - Version 21.5.33-3
+        - 539 Labels in total. Loc: 35 / Env: 13 / App: 368 / Role: 123
+        - Workloads: Managed: 5822 / Unmanaged: 1483 / Deleted: 0
+        - 0 IPlists in total.
+        - 0 RuleSets and 0 Rules.
+        """
+        stats = ""
+        stats += "{}- Version {}".format(padding, self.pce_version.generate_str_from_numbers()) + os.linesep
+
+        labels_str = ''
+        for dimension in self.LabelStore.label_types:
+            labels_str += " {}: {} /".format(dimension, self.LabelStore.count_labels(dimension))
+        # remove last ' /'
+        labels_str = labels_str[:-2]
+        stats += "{}- {} Labels in total. {}".\
+            format(padding,
+                   self.LabelStore.count_labels(),
+                   labels_str)
+
+        stats += os.linesep + "{}- Workloads: Managed: {} / Unmanaged: {} / Deleted: {}". \
+            format(padding,
+                   self.WorkloadStore.count_managed_workloads(),
+                   self.WorkloadStore.count_unmanaged_workloads(True),
+                   self.WorkloadStore.count_deleted_workloads())
+
+        stats += os.linesep + "{}- {} IPlists in total.". \
+            format(padding,
+                   self.IPListStore.count())
+
+        stats += os.linesep + "{}- {} RuleSets and {} Rules.". \
+            format(padding, self.RulesetStore.count_rulesets(), self.RulesetStore.count_rules())
+
+        return stats
+
+

illumio_pylo/Query.py

@@ -0,0 +1,331 @@
+import illumio_pylo as pylo
+
+
+def find_chars(text: str, start: int):
+    end = len(text)
+    return {"ending_parenthesis": text.find(")", start),
+            "opening_parenthesis": text.find(")", start),
+            "opening_squote": text.find("'", start),
+            }
+
+
+def find_first_punctuation(text: str, start: int):
+    cursor = start
+    while cursor < len(text):
+        char = text[cursor]
+        if char == ')':
+            return {'notfound': False, 'position': cursor, 'character': ')'}
+        if char == '(':
+            return {'notfound': False, 'position': cursor, 'character': '('}
+        if char == "'":
+            return {'notfound': False, 'position': cursor, 'character': "'"}
+
+        cursor += 1
+
+    return {'notfound': True}
+
+
+class get_block_response:
+    def __init__(self, length=None, operator=None, error=None):
+        self.length = length
+        self.operator = operator
+        self.error = error
+
+
+def get_block_until_binary_ops_quotes_enabled(data: str):
+
+    detected_quote = None
+
+    for pos in range(len(data)):
+
+        cur_2let = data[pos:pos+2].lower()
+        cur_3let = data[pos:pos+3].lower()
+
+        # print("DEBUG {}  {}".format(cur_2let, cur_3let))
+
+        if cur_2let == 'or':
+            if detected_quote is None:
+                return get_block_response(length=pos, operator='or')
+        elif cur_3let == 'and':
+            if detected_quote is None:
+                return get_block_response(length=pos, operator='and')
+
+        cur_char = data[pos]
+
+        if cur_char == "'":
+            if detected_quote is None:
+                detected_quote = cur_char
+            elif detected_quote == '"':
+                continue
+            else:
+                detected_quote = None
+
+        elif cur_char == '"':
+            if detected_quote is None:
+                detected_quote = cur_char
+            elif detected_quote == "'":
+                continue
+            else:
+                detected_quote = None
+
+    if detected_quote is None:
+        return get_block_response(length=len(data))
+
+    return get_block_response(error="some quotes {} were not closed in expression: {}".format(detected_quote, data))
+
+
+class Query:
+    def __init__(self, level=0):
+        self.level = level
+        self.subQueries = []  # type: list[pylo.Query]
+        self.raw_value = None
+
+    def parse(self, data: str):
+        padding = ''.rjust(self.level*3)
+        data_len = len(data)
+        self.raw_value = data
+
+        cursor = 0
+        current_block_start = 0
+        parenthesis_opened = []
+        blocks = []
+        reached_end = False
+
+        print(padding + 'Level {} parsing string "{}"'.format(self.level , data))
+
+        while cursor < len(data):
+            find_punctuation = find_first_punctuation(data, cursor)
+            if find_punctuation['notfound']:
+                if len(parenthesis_opened) > 0:
+                    raise pylo.PyloEx("Reached the end of string before closing parenthesis in block: {}".format(data[current_block_start-1:]))
+                blocks.append({'type': 'text', 'text': data[current_block_start:]})
+                reached_end = True
+                print(padding + "{}-{} REACHED END OF STRING".format(self.level, len(blocks)))
+                break
+
+            found_character = find_punctuation['character']
+            found_character_position = find_punctuation['position']
+
+            if found_character == "'":
+                find_next_quote = data.find("'", found_character_position+1)
+                if find_next_quote == -1:
+                    raise pylo.PyloEx("Cannot find a matching closing quote from position {} in text: {}".format(found_character_position, data[current_block_start:]))
+                cursor = find_next_quote+1
+                if cursor >= len(data):
+                    blocks.append({'type': 'text', 'text': data[current_block_start:]})
+                continue
+            elif found_character == ")":
+                if len(parenthesis_opened) < 1:
+                    raise pylo.PyloEx("Cannot find a matching opening parenthesis at position #{} in text: {}".format(found_character_position, data[current_block_start:]))
+                elif len(parenthesis_opened) == 1:
+                    parenthesis_opened.pop()
+                    blocks.append({'type': 'sub', 'text': data[current_block_start:found_character_position]})
+                    current_block_start = found_character_position + 1
+                    cursor = current_block_start
+                    print(padding + "{}-{} REACHED BLOCK-END C PARENTHESIS".format(self.level, len(blocks)))
+                    continue
+                else:
+                    parenthesis_opened.pop()
+                    cursor = found_character_position + 1
+                    print(padding + "{}-{} REACHED MID-BLOCK C PARENTHESIS".format(self.level, len(blocks)))
+                    continue
+            elif found_character == "(":
+                parenthesis_opened.append(found_character_position)
+                print(padding + "{}-{} FOUND OPENING P".format(self.level, len(blocks)))
+
+                if (found_character_position == 0 or found_character_position > current_block_start) and len(parenthesis_opened) == 1:
+                    if found_character_position != 0:
+                        blocks.append({'type': 'text', 'text': data[current_block_start:found_character_position]})
+                    current_block_start = found_character_position+1
+                    cursor = current_block_start
+                    print(padding + "{}-{} OPENING P WAS FIRST".format(self.level, len(blocks)))
+                    continue
+                else:
+                    print(padding + "{}-{} OPENING P WAS NOT FIRST".format(self.level, len(blocks)))
+
+                cursor = found_character_position + 1
+                continue
+
+            cursor += 1
+
+        if len(parenthesis_opened) > 0:
+            raise pylo.PyloEx("Reached the end of string before closing parenthesis in block: {}".format(
+                data[current_block_start-1:]))
+
+        print(padding + "* Query Level {} blocks:".format(self.level))
+        for block in blocks:
+            print(padding + "- {}: |{}|".format(block['type'], block['text']))
+
+        # clear empty blocks, they're just noise
+        cleared_blocks = []
+        for block in blocks:
+            if block['type'] == 'text':
+                block['text'] = block['text'].strip()
+                if len(block['text']) < 1:
+                    continue
+            cleared_blocks.append(block)
+
+        # now building operator blocks
+        operator_blocks = []
+
+        for block_number in range(len(blocks)):
+            block = blocks[block_number]
+            if block['type'] == 'sub':
+                new_sub_query = Query(self.level+1)
+                new_sub_query.parse(block['text'])
+                self.subQueries.append(new_sub_query)
+                new_block = {'type': 'query', 'query': new_sub_query}
+                operator_blocks.append(new_block)
+                continue
+
+            text = block['text']
+            found_filter_name = False
+            found_filter_operator = False
+
+            filter_name = None
+            filter_operator = None
+            find_filter_in_collection = None
+
+            while True:
+                text_len = len(text)
+                if text_len < 1:
+                    break
+                print(padding + "* Handling of text block '||{}||'".format(text))
+                first_word_end = text.find(' ')
+
+                if first_word_end < 0:
+                    first_word = text
+                    first_word_end = text_len
+                else:
+                    first_word = text[0:first_word_end]
+                first_word_lower = first_word.lower()
+                print(padding + "  - First word '{}'".format(first_word))
+
+                if first_word_lower == 'or' or first_word_lower == 'and':
+                    if found_filter_name:
+                        if not found_filter_operator:
+                            raise pylo.PyloEx("Found binary operator '{}' while filter '{}' was found but no operator provided in expression '{}'".format(first_word, filter_name, block['text']))
+                        if find_filter_in_collection.arguments is not None:
+                            raise pylo.PyloEx(
+                                "Found binary operator '{}' while filter '{}' with operator '{}' requires arguments in expression '{}'".format(
+                                    first_word, filter_name, filter_operator, block['text']))
+                        new_block = {'type': 'filter', 'filter': find_filter_in_collection, 'raw_arguments': None}
+                        operator_blocks.append(new_block)
+                        found_filter_name = False
+                        found_filter_operator = False
+
+                    new_block = {'type': 'binary_op', 'value': first_word_lower}
+                    operator_blocks.append(new_block)
+                elif found_filter_name and found_filter_operator:
+                    block_info = get_block_until_binary_ops_quotes_enabled(text)
+                    if block_info.error is not None:
+                        raise pylo.PyloEx(block_info.error)
+
+                    if block_info.length == 0:
+                        new_block = {'type': 'filter', 'filter': find_filter_in_collection, 'raw_arguments': None}
+                        operator_blocks.append(new_block)
+                        print(padding+"   - Found no argument (or empty)")
+                        raise pylo.PyloEx("This should never happen")
+                    else:
+                        new_block = {'type': 'filter', 'filter': find_filter_in_collection, 'raw_arguments': text[0:block_info.length].strip()}
+                        operator_blocks.append(new_block)
+                        print(padding + "   - Found argument ||{}|| stopped by {}".format(new_block['raw_arguments'], block_info.operator))
+
+                    found_filter_name = False
+                    found_filter_operator = False
+
+                    first_word_end = block_info.length - 1
+
+                elif not found_filter_name and not found_filter_operator:
+                    filter_name = first_word
+                    found_filter_name = True
+                    find_filter_in_collection = FilterCollections.workload_filters.get(filter_name)
+                    if find_filter_in_collection is None:
+                        raise pylo.PyloEx("Cannot find a filter named '{}' in expression '{}'".format(filter_name, block['text']))
+                elif found_filter_name and not found_filter_operator:
+                    filter_operator = first_word
+                    found_filter_operator = True
+                    find_filter_in_collection = find_filter_in_collection.get(filter_operator)
+                    if find_filter_in_collection is None:
+                        raise pylo.PyloEx("Cannot find a filter operator '{}' for filter named '{}' in expression '{}'".format(filter_operator, filter_name, block['text']))
+
+                if first_word_end >= text_len:
+                    break
+                text = text[first_word_end + 1:].strip()
+
+        # showing blocks to check how well we did
+        for block in operator_blocks:
+            print(block)
+
+        # todo: optimize/handle inverters
+
+    def execute_on_single_object(self, object):
+        if len(self.subQueries) == 1:
+            pass
+
+        return False
+
+
+class Filter:
+    def __init__(self, name: str, func, arguments=None):
+        self.name = name
+        self.function = func
+        self.arguments = arguments
+
+
+class WorkloadFilter(Filter):
+    def __init__(self, name: str, func, arguments = None):
+        Filter.__init__(self, name, func, arguments)
+
+    def do_filter(self, workload: pylo.Workload):
+        return self.function(workload)
+
+
+class FilterContext:
+    def __init__(self, argument, math_op=None):
+        self.math_op = math_op
+        self.argument = argument
+
+
+class FilterCollections:
+    workload_filters = {}  # type: dict[str,WorkloadFilter]
+
+    @staticmethod
+    def add_workload_filter(name: str, operator: str, func, arguments=None):
+        new_filter = WorkloadFilter(name, func)
+        if FilterCollections.workload_filters.get(name) is None:
+            FilterCollections.workload_filters[name.lower()] = {}
+
+        if FilterCollections.workload_filters[name.lower()].get(operator) is not None:
+            raise pylo.PyloEx("Filter named '{}' with operator '{}' is already defined".format(name, operator))
+        FilterCollections.workload_filters[name.lower()][operator.lower()] = new_filter
+
+
+def tmp_func(wkl: pylo.Workload, context: FilterContext):
+    return wkl.get_name() == context.argument
+
+
+FilterCollections.add_workload_filter('name', 'matches', tmp_func, 'string')
+
+
+def tmp_func(wkl: pylo.Workload, context: FilterContext):
+    if context.math_op == '>':
+        return wkl.count_references() > context.argument
+
+
+FilterCollections.add_workload_filter('reference.count', '<>=!', tmp_func, 'int')
+
+
+def tmp_func(wkl: pylo.Workload, context: FilterContext):
+    return context.argument in wkl.description
+
+
+FilterCollections.add_workload_filter('description', 'contains', tmp_func, 'string')
+
+
+
+
+
+
+
+

illumio_pylo/ReferenceTracker.py

@@ -0,0 +1,41 @@
+import illumio_pylo as pylo
+from illumio_pylo import log
+from .Helpers import *
+
+
+class ReferenceTracker:
+    def __init__(self):
+        self._references = {}  # type: dict[Referencer, Referencer]
+
+    def add_reference(self, ref: 'pylo.Referencer'):
+        self._references[ref] = ref
+
+    def remove_reference(self, ref: 'pylo.Referencer'):
+        index = self._references.get(ref)
+        if index is None:
+            raise Exception('Tried to unreference an object which is not actually referenced')
+        self._references.pop(ref)
+
+    def count_references(self):
+        return len(self._references)
+
+    def get_references(self):
+        return self._references.values()
+
+    def get_references_filter_by_class(self, classes):
+        matches = []
+        for obj in self._references.values():
+            if type(obj) in classes:
+                matches.append(obj)
+        return matches
+
+
+class Referencer:
+    def reference_name_changed(self):
+        raise Exception('not implemented')
+
+
+class Pathable:
+    def __init__(self):
+        self.name = ''
+