illumio-pylo 0.2.5__py3-none-any.whl

illumio_pylo/WorkloadStore.py

@@ -0,0 +1,289 @@
+from illumio_pylo import log, IP4Map, PyloEx, Workload, nice_json, Label, LabelGroup
+from .Helpers import *
+from .Organization import Organization
+from typing import Optional, List, Union, Set
+
+
+class WorkloadStore:
+
+    def __init__(self, owner: 'Organization'):
+        self.owner: Organization = owner
+        self.itemsByHRef: Dict[str, Workload] = {}
+
+    def load_workloads_from_json(self, json_list):
+        for json_item in json_list:
+            if 'name' not in json_item or 'href' not in json_item:
+                raise PyloEx("Cannot find 'value'/name or href for Workload in JSON:\n" + nice_json(json_item))
+
+            new_item_name = json_item['name']
+            new_item_href = json_item['href']
+
+            # Workload's name is None when it's provided by VEN through its hostname until it's manually overwritten
+            # (eventually) by someone. In such a case, you need to use hostname instead
+            if new_item_name is None:
+                if 'hostname' not in json_item:
+                    raise PyloEx("Cannot find 'value'/hostname in JSON:\n" + nice_json(json_item))
+                new_item_name = json_item['hostname']
+
+            new_item = Workload(new_item_name, new_item_href, self)
+            new_item.load_from_json(json_item)
+
+            if new_item_href in self.itemsByHRef:
+                raise PyloEx("A Workload with href '%s' already exists in the table", new_item_href)
+
+            self.itemsByHRef[new_item_href] = new_item
+
+            log.debug("Found Workload '%s' with href '%s'", new_item_name, new_item_href)
+
+    def find_by_href_or_die(self, href: str) -> 'Workload':
+        """
+        Find a Workload from its HREF, throw an Exception if not found
+
+        :param href: the HREF you are looking for
+        :return:
+        :raises:
+            PyloEx: if no Workload matching provided HREF
+        """
+        find_object = self.itemsByHRef.get(href)
+        if find_object is None:
+            raise PyloEx("Workload with HREF '%s' was not found" % href)
+
+        return find_object
+
+    def find_by_href_or_create_tmp(self, href: str, tmp_wkl_name: str) -> 'Workload':
+        """
+        Find a Workload from its HREF, creates a fake temporary one if not found. *Reserved for developers*
+
+        :param href: the HREF you are looking for
+        :return:
+        """
+        find_object = self.itemsByHRef.get(href)
+        if find_object is not None:
+            return find_object
+
+        new_tmp_item = Workload(tmp_wkl_name, href, self)
+        new_tmp_item.deleted = True
+        new_tmp_item.temporary = True
+
+        self.itemsByHRef[href] = new_tmp_item
+
+        return new_tmp_item
+
+    def find_workloads_matching_label(self, label: 'Label') -> Dict[str, 'Workload']:
+        """
+        Find all Workloads which are using a specific Label.
+
+        :param label: Label you want to match on
+        :return: a dictionary of all matching Workloads using their HREF as key
+        """
+        result = {}
+
+        for href, workload in self.itemsByHRef.items():
+            if workload.is_using_label(label):
+                result[href] = workload
+
+        return result
+
+    def find_workloads_matching_all_labels(self, labels: Union[List[Label|LabelGroup], Dict[str,Label|LabelGroup]])\
+            -> Dict[str, 'Workload']:
+        """
+        Find all Workloads which are using all the Labels from a specified list/dict. Note that labels will be ordered by type
+        and workloads will need to match 1 Label of each specified. LabelGroups will be expanded.
+        If one of the labels is None, it will be ignored
+
+        :param labels: list of Labels you want to match on
+        :return: a dictionary of all matching Workloads using their HREF as key
+        """
+        unique_labels: Set[Union[Label, LabelGroup]] = set()
+
+        if isinstance(labels, list):
+            for label in labels:
+                if label is None:
+                    pylo.log.debug("Label is None, skipping")
+                    continue
+                if isinstance(label, LabelGroup):
+                    unique_labels.update(label.expand_nested_to_dict_by_href().values())
+                else:
+                    unique_labels.add(label)
+        else:
+            for label in labels.values():
+                if label is None:
+                    pylo.log.warn("Label is None, skipping")
+                    continue
+                if isinstance(label, LabelGroup):
+                    unique_labels.update(label.expand_nested_to_dict_by_href().values())
+                else:
+                    unique_labels.add(label)
+
+        labels_by_type = pylo.LabelStore.Utils.list_to_dict_by_type(unique_labels)
+
+        result = {}
+
+        for workload in self.itemsByHRef.values():
+            workload_is_a_match = True
+            for label_type, labels_to_find in labels_by_type.items():
+                workload_label = workload.get_label(label_type)
+                if workload_label is None or workload_label not in labels_to_find:
+                    workload_is_a_match = False
+                    break
+            if workload_is_a_match:
+                result[workload.href] = workload
+
+        return result
+
+
+    def find_workload_matching_forced_name(self, name: str, case_sensitive: bool = True, strip_fqdn: bool = False) -> Optional[Workload]:
+        """
+        Find a Workload based on its name (case-sensitive). Beware that if several are matching, only the first one will be returned
+
+        :param name: the name string you are looking for
+        :param case_sensitive: make it a case-sensitive search or not
+        :param strip_fqdn: remove the fqdn part of the hostname
+        :return: the Workload it found, None otherwise
+        """
+        if not case_sensitive:
+            name = name.lower()
+
+        for workload in self.itemsByHRef.values():
+            wkl_name = workload.forced_name
+            if strip_fqdn:
+                wkl_name = Workload.static_name_stripped_fqdn(wkl_name)
+            if case_sensitive:
+                if wkl_name == name:
+                    return workload
+            else:
+                if wkl_name.lower() == name:
+                    return workload
+
+        return None
+
+    def find_workload_matching_hostname(self, name: str, case_sensitive: bool = True, strip_fqdn: bool = False, fall_back_to_name: bool = False) -> Optional[Workload]:
+        """
+        Find a workload based on its hostname.Beware that if several are matching, only the first one will be returned
+
+        :param name: the name string you are looking for
+        :param case_sensitive: make it a case-sensitive search or not
+        :param strip_fqdn: remove the fqdn part of the hostname
+        :param fall_back_to_name: if True, will fall back to the forced name if no hostname is found
+        :return: the Workload it found, None otherwise
+        """
+        if not case_sensitive:
+            name = name.lower()
+
+        for workload in self.itemsByHRef.values():
+            wkl_name = workload.hostname
+            if wkl_name is None:
+                if fall_back_to_name:
+                    wkl_name = workload.forced_name
+                    if wkl_name is None:
+                        continue
+                else:
+                    continue
+            if strip_fqdn:
+                wkl_name = Workload.static_name_stripped_fqdn(wkl_name)
+            if case_sensitive:
+                if wkl_name == name:
+                    return workload
+            else:
+                if wkl_name.lower() == name:
+                    return workload
+
+        return None
+
+    def find_all_workloads_matching_hostname(self, name: str, case_sensitive: bool = True, strip_fqdn: bool = False, fall_back_to_name: bool = False) -> List[Workload]:
+        """
+        Find all workloads based on their hostnames.
+        :param name: the name string you are looking for
+        :param case_sensitive: make it a case-sensitive search or not
+        :param strip_fqdn: remove the fqdn part of the hostname
+        :param fall_back_to_name: if True, will fall back to the forced name if no hostname is found
+        :return: list of matching Workloads
+        """
+        result = []
+
+        if not case_sensitive:
+            name = name.lower()
+
+        for workload in self.itemsByHRef.values():
+            wkl_name = workload.hostname
+            if wkl_name is None:
+                if fall_back_to_name:
+                    wkl_name = workload.forced_name
+                    if wkl_name is None:
+                        continue
+                else:
+                    continue
+            if strip_fqdn:
+                wkl_name = Workload.static_name_stripped_fqdn(wkl_name)
+            if case_sensitive:
+                if wkl_name == name:
+                    result.append(workload)
+            else:
+                if wkl_name.lower() == name:
+                    result.append(workload)
+
+        return result
+
+    def count_workloads(self) -> int:
+        return len(self.itemsByHRef)
+
+    def count_managed_workloads(self) -> int:
+        count = 0
+
+        for item in self.itemsByHRef.values():
+            if not item.unmanaged and not item.deleted:
+                count += 1
+
+        return count
+
+    def get_managed_workloads_list(self) -> List['Workload']:
+        """
+        Get a list of all managed workloads
+        :return:
+        """
+        results = []
+        for item in self.itemsByHRef.values():
+            if not item.unmanaged:
+                results.append(item)
+
+        return results
+
+    def get_managed_workloads_dict_by_href(self) -> Dict[str, 'Workload']:
+        """
+        Get a dictionary of all managed workloads using their HREF as key
+        :return:
+        """
+        results = {}
+        for item in self.itemsByHRef.values():
+            if not item.unmanaged:
+                results[item.href] = item
+
+        return results
+
+    def count_deleted_workloads(self) -> int:
+        count = 0
+        for item in self.itemsByHRef.values():
+            if item.deleted:
+                count += 1
+                #print(item.href)
+
+        return count
+
+    def count_unmanaged_workloads(self, if_not_deleted=False) -> int:
+        count = 0
+
+        for item in self.itemsByHRef.values():
+            if item.unmanaged and (not if_not_deleted or (if_not_deleted and not item.deleted)):
+                count += 1
+
+        return count
+
+    @property
+    def workloads(self) -> List['Workload']:
+        """
+        Get a list of all workloads
+        :return:
+        """
+        return list(self.itemsByHRef.values())
+
+

illumio_pylo/__init__.py

@@ -0,0 +1,82 @@
+import os
+import sys
+from typing import Callable
+
+from .tmp import *
+from .Helpers import *
+
+from .Exception import PyloEx, PyloApiEx, PyloApiTooManyRequestsEx, PyloApiUnexpectedSyntax, PyloObjectNotFound, PyloApiRequestForbiddenEx
+from .SoftwareVersion import SoftwareVersion
+from .IPMap import IP4Map
+from .ReferenceTracker import ReferenceTracker, Referencer, Pathable
+from .API.APIConnector import APIConnector, ObjectTypes
+from .API.RuleSearchQuery import RuleSearchQuery, RuleSearchQueryResolvedResultSet
+from .API.ClusterHealth import ClusterHealth
+from .API.Explorer import ExplorerResultSetV1, RuleCoverageQueryManager, ExplorerFilterSetV1, ExplorerQuery
+from .API.AuditLog import AuditLogQuery, AuditLogQueryResultSet, AuditLogFilterSet
+from .API.CredentialsManager import get_credentials_from_file
+from .LabelCommon import LabelCommon
+from .Label import Label
+from .LabelGroup import LabelGroup
+from .LabelStore import LabelStore, label_type_app, label_type_env, label_type_loc, label_type_role
+from .IPList import IPList, IPListStore
+from .AgentStore import AgentStore, VENAgent
+from .Workload import Workload, WorkloadInterface
+from .WorkloadStore import WorkloadStore
+from .VirtualService import VirtualService
+from .VirtualServiceStore import VirtualServiceStore
+from .Service import Service, ServiceStore, PortMap, ServiceEntry
+from .Rule import Rule, RuleServiceContainer, RuleSecurityPrincipalContainer, DirectServiceInRule, RuleHostContainer, RuleActorsAcceptableTypes
+from .Ruleset import Ruleset, RulesetScope, RulesetScopeEntry
+from .RulesetStore import RulesetStore
+from .SecurityPrincipal import SecurityPrincipal, SecurityPrincipalStore
+from .Organization import Organization
+from .Query import Query
+
+
+def get_organization(fqdn: str, port: int, api_user: str, api_key: str,
+                     organization_id: int, verify_ssl: bool = True,
+                     list_of_objects_to_load: Optional[List['pylo.ObjectTypes']] = None,
+                     include_deleted_workloads: bool = False) -> Organization:
+    """
+    Load an organization from the API with parameters provided as arguments.
+    """
+    api = APIConnector(fqdn=fqdn, port=port, apiuser=api_user, apikey=api_key, org_id=organization_id,
+                       skip_ssl_cert_check=not verify_ssl)
+    org = Organization(1)
+    org.load_from_api(api, include_deleted_workloads=include_deleted_workloads,
+                      list_of_objects_to_load=list_of_objects_to_load)
+
+    return org
+
+
+def get_organization_using_credential_file(fqdn_or_profile_name: str = None,
+                                           credential_file: str = None,
+                                           list_of_objects_to_load: Optional[List['pylo.ObjectTypes']] = None,
+                                           include_deleted_workloads: bool = False,
+                                           callback_api_objects_downloaded: Callable = None) -> Organization:
+    """
+    Credentials files will be looked for in the following order:
+    1. The path provided in the credential_file argument
+    2. The path provided in the Pylo_CREDENTIAL_FILE environment variable
+    3. The path ~/.pylo/credentials.json
+    4. Current working directory credentials.json
+    :param fqdn_or_profile_name:
+    :param credential_file:
+    :param list_of_objects_to_load:
+    :param include_deleted_workloads:
+    :param callback_api_objects_downloaded: callback function that will be called after each API has finished downloading all objects
+    :return:
+    """
+    return Organization.get_from_api_using_credential_file(fqdn_or_profile_name=fqdn_or_profile_name,
+                                                           credential_file=credential_file,
+                                                           list_of_objects_to_load=list_of_objects_to_load,
+                                                           include_deleted_workloads=include_deleted_workloads,
+                                                           callback_api_objects_downloaded=callback_api_objects_downloaded)
+
+
+
+ignoreWorkloadsWithSameName = True
+
+objectNotFound = object()
+

illumio_pylo/cli/NativeParsers.py

@@ -0,0 +1,96 @@
+import argparse
+from typing import Optional, List, Dict
+
+import illumio_pylo as pylo
+
+
+class BaseParser:
+    _action: Optional[argparse.Action] = None
+
+    def fill_parser(self, parser: argparse.ArgumentParser):
+        raise NotImplementedError
+
+    def execute(self, args: str | int, org: 'pylo.Organization', padding: str = '') -> Optional[List['pylo.Label']]:
+        raise NotImplementedError
+
+    def get_arg_name(self) -> str:
+        return self._action.dest
+
+
+class LabelParser(BaseParser):
+    def __init__(self, action_name: str, action_short_name: Optional[str], label_type: Optional[str] = None, is_required: bool = True, allow_multiple: bool = False, help_text: Optional[str] = None):
+        self.action_name = action_name
+        self.action_short_name = action_short_name
+        self.label_type = label_type
+        self.is_required = is_required
+        self.allow_multiple = allow_multiple
+        self.results: Optional[List['pylo.Label']] = None
+        self.results_as_dict_by_href: Dict[str, 'pylo.Label'] = {}
+        self.help_text = help_text
+
+    def fill_parser(self, parser: argparse.ArgumentParser):
+
+        help_text = self.help_text
+        if help_text is None:
+            if self.allow_multiple:
+                if self.label_type is None:
+                    help_text = f"Filter by label name. Multiple labels can be specified by separating them with a comma."
+                else:
+                    help_text = f"Filter by label name of type '{self.label_type}'. Multiple labels can be specified by separating them with a comma."
+            else:
+                if self.label_type is None:
+                    help_text = f"Filter by label name."
+                else:
+                    help_text = f"Filter by label name of type '{self.label_type}'."
+
+        if self.action_short_name is None:
+            self._action = parser.add_argument(self.action_name, type=str,
+                                               required=self.is_required, help=help_text)
+        else:
+            self._action = parser.add_argument(self.action_short_name, self.action_name, type=str,
+                                               required=self.is_required, help=help_text)
+
+    def execute(self, args: str, org: 'pylo.Organization', padding: str = ''):
+        print(f"{padding}{self.action_name}:", end="")
+        if args is None:
+            print(" None")
+            return
+
+        if self.allow_multiple:
+            label_names = args.split(",")
+        else:
+            label_names = [args]
+
+        missing_labels: List[str] = []
+
+        label_objects = org.LabelStore.find_label_by_name(label_names, label_type=self.label_type, missing_labels_names=missing_labels)
+        if len(missing_labels) > 0:
+            raise pylo.PyloEx(f"Could not find labels: {pylo.string_list_to_text(missing_labels)}, please check their spelling, case and type.")
+
+        # just in case make sure it's a list of unique labels
+        self.results = list(set(label_objects))
+        self.results_as_dict_by_href = {label.href: label for label in self.results}
+        print(f" {pylo.string_list_to_text(self.results)}")
+
+
+    def filter_workloads_matching_labels(self, workloads: List['pylo.Workload']) -> List['pylo.Workload']:
+        if self.results is None:
+            return workloads
+
+        # we must group Labels by their type first
+        labels_dict_by_type: Dict[str, List[pylo.Label]] = {}
+        for label in self.results:
+            if labels_dict_by_type.get(label.type) is None:
+                labels_dict_by_type[label.type] = [label]
+            else:
+                labels_dict_by_type[label.type].append(label)
+
+        # now we can filter the workloads, they must match at least one Label per type
+        filtered_workloads = []
+        for workload in workloads:
+            for label_type in labels_dict_by_type:
+                if workload.get_label(label_type) in labels_dict_by_type[label_type]:
+                    filtered_workloads.append(workload)
+
+        return filtered_workloads
+

illumio_pylo/cli/__init__.py

@@ -0,0 +1,134 @@
+import os
+from typing import Optional, Dict
+
+import sys
+import argparse
+from .NativeParsers import BaseParser
+
+# in case user wants to run this utility while having a version of pylo already installed
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+
+import illumio_pylo as pylo
+from illumio_pylo.cli import commands
+
+
+def run(forced_command_name: Optional[str] = None):
+
+    def add_native_parser_to_argparse(parser: argparse.ArgumentParser, native_parsers: object):
+        # each property of the native parser is an extension of BaseParser, we need to iterate over them and add them to the argparse parser
+        for attr_name in dir(native_parsers):
+            attr = getattr(native_parsers, attr_name)
+            if isinstance(attr, BaseParser):
+                attr.fill_parser(parser)
+
+    def execute_native_parsers(args: Dict, org: pylo.Organization, native_parsers: object):
+        first_native_parser_found = False
+
+        for attr_name in dir(native_parsers):
+            attr = getattr(native_parsers, attr_name)
+            if isinstance(attr, BaseParser):
+                if first_native_parser_found is False:
+                    first_native_parser_found = True
+                    print(" * Native CLI arguments parsing...")
+                attr.execute(args[attr.get_arg_name()], org, padding='    ')
+
+    parser = argparse.ArgumentParser(description='TODO LATER')
+    parser.add_argument('--pce', type=str, required=False,
+                        help='hostname of the PCE')
+    parser.add_argument('--debug', action='store_true',
+                        help='Enables extra debugging output in Pylo framework')
+    parser.add_argument('--use-cache', action='store_true',
+                        help='For developers only')
+
+    selected_command = None
+
+    if forced_command_name is None:
+        sub_parsers = parser.add_subparsers(dest='command', required=True)
+        for command in commands.available_commands.values():
+            sub_parser = sub_parsers.add_parser(command.name, help='')
+            command.fill_parser(sub_parser)
+            if command.native_parsers is not None:
+                add_native_parser_to_argparse(sub_parser, command.native_parsers)
+    else:
+        for command in commands.available_commands.values():
+            if forced_command_name is not None and command.name != forced_command_name:
+                continue
+            command.fill_parser(parser)
+            if command.native_parsers is not None:
+                add_native_parser_to_argparse(parser, command.native_parsers)
+            selected_command = command
+
+    args = vars(parser.parse_args())
+
+    if args['debug']:
+        pylo.log_set_debug()
+
+    credential_profile_name = args['pce']
+    settings_use_cache = args['use_cache']
+    org: Optional[pylo.Organization] = None
+
+    # We are getting the command object associated to the command name if it was not already set (via forced_command_name)
+    if selected_command is None:
+        selected_command = commands.available_commands[args['command']]
+        if selected_command is None:
+            raise pylo.PyloEx("Cannot find command named '{}'".format(args['command']))
+
+    connector: Optional[pylo.APIConnector] = None
+    config_data = None
+
+    if not selected_command.credentials_manager_mode:
+        # credential_profile_name is required for all commands except the credential manager
+        if credential_profile_name is None:
+            raise pylo.PyloEx("The --pce argument is required for this command")
+        if settings_use_cache:
+            print(" * Loading objects from cached PCE '{}' data... ".format(credential_profile_name), end="", flush=True)
+            org = pylo.Organization.get_from_cache_file(credential_profile_name)
+            print("OK!")
+            connector = pylo.APIConnector.create_from_credentials_in_file(credential_profile_name, request_if_missing=False)
+            if connector is not None:
+                org.connector = connector
+        else:
+            print(" * Looking for PCE/profile '{}' credentials... ".format(credential_profile_name), end="", flush=True)
+            connector = pylo.APIConnector.create_from_credentials_in_file(credential_profile_name, request_if_missing=True)
+            print("OK!")
+
+            print(" * Downloading PCE objects from API... ".format(credential_profile_name), end="", flush=True)
+            config_data = connector.get_pce_objects(list_of_objects_to_load=selected_command.load_specific_objects_only)
+            print("OK!")
+
+            org = pylo.Organization(1)
+            org.connector = connector
+
+            if not selected_command.skip_pce_config_loading:
+                print(" * Loading objects from PCE '{}' via API... ".format(credential_profile_name), end="", flush=True)
+                org.pce_version = connector.get_software_version()
+                org.load_from_json(config_data, list_of_objects_to_load=selected_command.load_specific_objects_only)
+                print("OK!")
+
+        print()
+        if not selected_command.skip_pce_config_loading:
+            print(" * PCE statistics: ")
+            print(org.stats_to_str(padding='    '))
+
+            print(flush=True)
+
+    print("**** {} UTILITY ****".format(selected_command.name.upper()), flush=True)
+    if selected_command.native_parsers is None:
+        native_parsers = None
+    else:
+        native_parsers = selected_command.native_parsers
+        execute_native_parsers(args, org, native_parsers)
+
+    if native_parsers is not None:
+        commands.available_commands[selected_command.name].main(args, org=org, config_data=config_data, connector=connector, native_parsers=native_parsers, pce_cache_was_used=settings_use_cache)
+    else:
+        commands.available_commands[selected_command.name].main(args, org=org, config_data=config_data, connector=connector, pce_cache_was_used=settings_use_cache)
+
+    print()
+    print("**** END OF {} UTILITY ****".format(selected_command.name.upper()))
+    print()
+
+
+if __name__ == "__main__":
+    run()

illumio_pylo/cli/__main__.py

@@ -0,0 +1,10 @@
+import sys
+import os
+
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+
+from illumio_pylo.cli import run
+
+if __name__ == "__main__":
+    run()

illumio_pylo/cli/commands/__init__.py

@@ -0,0 +1,32 @@
+from typing import Dict, Optional, List
+
+
+class Command:
+    def __init__(self, name: str, main_func, parser_func, load_specific_objects_only: Optional[List[str]] = None,
+                 skip_pce_config_loading: bool = False,
+                 native_parsers_as_class: Optional = None,
+                 credentials_manager_mode: bool = False):
+        self.name: str = name
+        self.main = main_func
+        self.fill_parser = parser_func
+        self.load_specific_objects_only: Optional[List[str]] = load_specific_objects_only
+        self.skip_pce_config_loading = skip_pce_config_loading
+        self.native_parsers = native_parsers_as_class
+        self.credentials_manager_mode = credentials_manager_mode
+        available_commands[name] = self
+
+
+available_commands: Dict[str, Command] = {}
+
+from .ruleset_export import command_object
+from .workload_used_in_rule_finder import command_object
+from .workload_relabeler import command_object
+from .ven_duplicate_remover import command_object
+from .workload_export import command_object
+from .iplist_import_from_file import command_object
+from .update_pce_objects_cache import command_object
+from .ven_upgrader import command_object
+from .workload_import import command_object
+from .ven_idle_to_visibility import command_object
+from .workload_reset_names_to_null import command_object
+from .credential_manager import command_object