illumio-pylo 0.2.5__py3-none-any.whl

illumio_pylo/tmp.py

@@ -0,0 +1,104 @@
+import logging
+import illumio_pylo as pylo
+
+log = logging.getLogger('Pylo')
+
+
+def init_logger():
+    console_logger = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    console_logger.setFormatter(formatter)
+    log.addHandler(console_logger)
+
+
+init_logger()
+
+
+def log_set_debug():
+    log.setLevel(logging.DEBUG)
+
+
+def log_set_info():
+    log.setLevel(logging.INFO)
+
+
+def log_set_warning():
+    log.setLevel(logging.WARNING)
+
+
+def get_logger():
+    return log
+
+
+def find_connector_or_die(obj):
+    connector = obj.__dict__.get('connector')  # type: pylo.APIConnector
+    if connector is None:
+        owner = obj.__dict__.get('owner')
+        if owner is None:
+            raise Exception("Could not find a Connector object")
+        return find_connector_or_die(owner)
+
+    return connector
+
+
+class IDTranslationTable:
+    """docstring fo ID_TranslationTable."""
+
+    def __init__(self):
+        self.OldToNew = {}
+        self.NewToOld = {}
+
+        self.sourcesSorting = {}
+        self.destSorting = {}
+
+    def add_source(self, key, value):
+        if key in self.sourcesSorting:
+            raise Exception("Duplicate key '%s'" % key)
+        self.sourcesSorting[key] = value
+
+        find = self.destSorting.get(key)
+        if find is None:
+            return
+
+        self.OldToNew[value] = find
+        self.NewToOld[find] = value
+
+    def add_destination(self, key, value):
+        if key in self.destSorting:
+            raise Exception("Duplicate key '%s'" % key)
+        self.destSorting[key] = value
+
+        find = self.sourcesSorting.get(key)
+        if find is None:
+            return
+
+        self.OldToNew[find] = value
+        self.NewToOld[value] = find
+
+    def find_new(self, old):
+        return self.OldToNew.get(old)
+
+    def find_new_or_die(self, old):
+        ret = self.OldToNew.get(old)
+        if ret is None:
+            raise Exception("Cannot find a match in the table for key '{}'".format(old))
+        return ret
+
+    def find_old(self, new):
+        return self.NewToOld.get(new)
+
+    def find_old_or_die(self, new):
+        ret = self.NewToOld.get(new)
+        if ret is None:
+            raise Exception("Cannot find a match in the table for key {]".format(new))
+        return ret
+
+    def stats_to_str(self, padding = ''):
+        msg = '{}source entries:{}\n{}destination entries:{}\n{}translation entries:{}'.\
+            format( padding, len(self.sourcesSorting),
+                    padding, len(self.destSorting),
+                    padding, len(self.NewToOld))
+        return msg
+
+    def keys_old(self):
+        return

illumio_pylo/utilities/cli.py

@@ -0,0 +1,10 @@
+# this is here only for the convenience of building binaries
+
+import os
+import sys
+
+# in case user wants to run this utility while having a version of pylo already installed
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+import illumio_pylo.cli
+
+illumio_pylo.cli.run()

illumio_pylo/utilities/credentials.example.json

@@ -0,0 +1,20 @@
+[
+    {
+        "name": "prod",
+        "hostname": "pce-prod.company.com",
+        "port": 9443,
+        "api_user": "apiuser_xxxxx",
+        "api_key": "xxxxxxxx",
+        "org_id": 1,
+        "verify_ssl": true
+    },
+    {
+        "name": "dev",
+        "hostname": "pce-dev.company.com",
+        "port": 8443,
+        "api_user": "apiuser_xxxxx",
+        "api_key": "xxxxxxxx",
+        "org_id": 3,
+        "verify_ssl": false
+    }
+]

illumio_pylo/utilities/explorer_report_exporter.py

@@ -0,0 +1,86 @@
+import os
+import sys
+import argparse
+import math
+from datetime import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+import illumio_pylo as pylo
+
+
+# <editor-fold desc="Argparse stuff">
+parser = argparse.ArgumentParser(description='TODO LATER', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('--pce', type=str, required=True,
+                    help='hostname of the PCE')
+parser.add_argument('--debug', '-d', action='store_true',
+                    help='extra debugging messages for developers')
+
+parser.add_argument('--consumer-labels', '-cl', type=str, nargs='+', required=False, default=False,
+                    help='extra debugging messages for developers')
+# </editor-fold>
+
+
+args = vars(parser.parse_args())
+
+if args['debug']:
+    pylo.log_set_debug()
+
+
+hostname = args['pce']
+
+
+now = datetime.now()
+report_file = 'explorer-results_{}.csv'.format(now.strftime("%Y%m%d-%H%M%S"))
+report_file_excel = 'explorer-results_{}.xlsx'.format(now.strftime("%Y%m%d-%H%M%S"))
+
+
+csv_report_headers = ['name', 'hostname', 'role', 'app', 'env', 'loc', 'href']
+csv_report = pylo.ArrayToExport(csv_report_headers)
+
+
+# <editor-fold desc="PCE Configuration Download and Parsing">
+org = pylo.Organization(1)
+fake_config = pylo.Organization.create_fake_empty_config()
+
+print(" * Looking for credentials for PCE '{}'... ".format(hostname), end="", flush=True)
+connector = pylo.APIConnector.create_from_credentials_in_file(hostname, request_if_missing=True)
+print("OK!")
+
+print(" * Downloading Labels listing from the PCE... ", end="", flush=True)
+fake_config['labels'] = connector.objects_label_get()
+print("OK!")
+
+print(" * Parsing PCE data ... ", end="", flush=True)
+org.pce_version = connector.version
+org.connector = connector
+org.load_from_json(fake_config)
+print("OK!")
+
+# </editor-fold>
+
+
+print(" - Now building query parameters:")
+if args['consumer_labels'] is not False:
+    print("   - Consumer labels were provided:")
+else:
+    print("   - No Consumer label provided")
+
+explorer_filters = connector.ExplorerFilterSetV1(max_results=2)
+
+print(" - Querying PCE...", end='', flush=True)
+search_results = connector.explorer_search(explorer_filters)
+print(" OK!")
+print(pylo.nice_json(search_results))
+
+
+print()
+print(" * Writing report file '{}' ... ".format(report_file), end='', flush=True)
+csv_report.write_to_csv(report_file)
+print("DONE")
+print(" * Writing report file '{}' ... ".format(report_file_excel), end='', flush=True)
+csv_report.write_to_excel(report_file_excel)
+print("DONE")
+
+if csv_report.lines_count() < 1:
+    print("\n** WARNING: no entry matched your filters so reports are empty !\n")
+

illumio_pylo/utilities/health_monitoring.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import argparse
+import math
+from datetime import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+import illumio_pylo as pylo
+
+
+# <editor-fold desc="Argparse stuff">
+parser = argparse.ArgumentParser(description='TODO LATER', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('--pce', type=str, required=True,
+                    help='hostname of the PCE')
+parser.add_argument('--pick-report-hostname', '-p', type=str, required=False, default=None,
+                    help='if script cannot guess which PCE hostname you were looking for, use this option to force it')
+parser.add_argument('--debug', action='store_true',
+                    help='extra debugging messages for developers')
+# </editor-fold>
+
+args = vars(parser.parse_args())
+
+if args['debug']:
+    pylo.log_set_debug()
+
+hostname = args['pce']
+
+pylo.log.debug(" * Looking for credentials for PCE '{}'... ".format(hostname))
+connector = pylo.APIConnector.create_from_credentials_in_file(hostname, request_if_missing=True)
+pylo.log.debug("OK!")
+
+pylo.log.debug(" * Getting cluster health details... ")
+health_collection = connector.cluster_health_get(return_object=True)
+pylo.log.debug("OK!")
+
+picked_hostname = connector.hostname
+if args['pick_report_hostname'] is not None:
+    picked_hostname = args['pick_report_hostname']
+
+health_data = health_collection.get(picked_hostname)
+
+if health_data is None:
+    pylo.log.error("No report for hostname '{}' was found in the collection! The following were available: {}.\n"
+    "You can use --pick-report-hostname to force usage of specific one.".format(picked_hostname, pylo.string_list_to_text(health_collection.keys())))
+    sys.exit(1)
+
+pylo.log.debug("Status details:\n{}\n".format(health_data.to_string()))
+
+warning_messages = []
+error_messages = []
+
+if health_data.status_is_warning():
+    warning_messages.append("Cluster has reported global 'warning' status read the entire report or issues troubleshooting command to get more details")
+elif health_data.status_is_error():
+    warning_messages.append("Cluster has reported global 'error' status read the entire report or issues troubleshooting command to get more details")
+
+broken_nodes = {}
+working_nodes = {}
+
+for node in health_data.nodes_dict.values():
+    if node.is_offline_or_unreachable():
+        broken_nodes[node] = node
+        warning_messages.append("Node {}/{}/IP:{} is offline or not reachable by other members".format(node.name, node.type, node.ip_address))
+    else:
+        working_nodes[node] = node
+
+for node in working_nodes.values():
+    troubled_services = node.get_troubled_services()
+    if len(troubled_services) > 0:
+        broken_nodes[node] = node
+        warning_messages.append("Node '{}'/{}/IP:{} has several non-functional services: {}".format(node.name, node.type, node.ip_address, pylo.string_list_to_text(troubled_services)))
+
+
+data1_is_broken = False
+data0_is_broken = False
+for node in broken_nodes:
+    if node.type == 'data0':
+        data0_is_broken = True
+    if node.type == 'data1':
+        data1_is_broken = True
+
+if data1_is_broken or data0_is_broken:
+    error_messages.append("Data1 or Data0 is down so Database has no resiliency anymore, please fix this situation as soon as possible.")
+
+report = {'warning_messages': warning_messages, 'error_messages': error_messages}
+
+if len(error_messages) > 0:
+    report['general_status'] = "error"
+    exit_code = 1
+elif len(warning_messages) > 0:
+    report['general_status'] = "warning"
+    exit_code = 2
+else:
+    report['general_status'] = "normal"
+    exit_code = 0
+
+print("{}".format(pylo.nice_json(report)))
+
+exit(exit_code)
+
+
+

illumio_pylo/utilities/iplist_analyzer.py

@@ -0,0 +1,148 @@
+import os
+import sys
+import argparse
+import math
+from datetime import datetime
+from typing import Dict, List, Any
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+import illumio_pylo as pylo
+
+
+# <editor-fold desc="Argparse stuff">
+parser = argparse.ArgumentParser(description='TODO LATER', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('--pce', type=str, required=True,
+                    help='hostname of the PCE')
+parser.add_argument('--dev-use-cache', type=bool, nargs='?', required=False, default=False, const=True,
+                    help='For developers only')
+parser.add_argument('--debug', '-d', type=bool, nargs='?', required=False, default=False, const=True,
+                    help='extra debugging messages for developers')
+parser.add_argument('--verbose', '-v', action='store_true',
+                    help='')
+
+# </editor-fold>
+
+
+args = vars(parser.parse_args())
+
+if args['debug']:
+    pylo.log_set_debug()
+
+
+hostname = args['pce']
+verbose = args['verbose']
+use_cached_config = args['dev_use_cache']
+
+
+now = datetime.now()
+report_file = 'iplist-analyzer_{}.csv'.format(now.strftime("%Y%m%d-%H%M%S"))
+report_file_excel = 'iplist-analyzer_{}.xlsx'.format(now.strftime("%Y%m%d-%H%M%S"))
+
+filter_csv_expected_fields = []
+filter_data = None
+
+
+csv_report_headers = ['name', 'members', 'ip4_mapping', 'ip4_count', 'ip4_uncovered_count', 'covered_workloads_count',
+                      'covered_workloads_list', 'covered_workloads_appgroups', 'href']
+
+csv_report = pylo.ArrayToExport(csv_report_headers)
+
+
+# <editor-fold desc="PCE Configuration Download and Parsing">
+org = pylo.Organization(1)
+if use_cached_config:
+    print(" * Loading cached PCE data from disk...")
+    org.load_from_cache_or_saved_credentials(hostname)
+    print("OK!")
+else:
+    print(" * Looking for credentials for PCE '{}'... ".format(hostname), end="", flush=True)
+    connector = pylo.APIConnector.create_from_credentials_in_file(hostname, request_if_missing=True)
+    print("OK!")
+
+    print(" * Downloading objects data from the PCE... ", end="", flush=True)
+    config = connector.get_pce_objects()
+    print("OK!")
+
+    print(" * Parsing PCE objects data ... ", end="", flush=True)
+    org.pce_version = connector.version
+    org.connector = connector
+    org.load_from_json(config)
+    print("OK!")
+
+print(" * PCE data statistics:\n{}".format(org.stats_to_str(padding='    ')))
+# </editor-fold>
+
+
+def add_iplist_to_report(iplist: pylo.IPList):
+
+    appgroup_tracker: Dict[str, bool] = {}
+
+    print("  - {}/{}".format(iplist.name, iplist.href))
+
+    ip_map = iplist.get_ip4map()
+    #print(ip_map.print_to_std(padding="     "))
+
+    new_row = {
+        'name': iplist.name,
+        'href': iplist.href,
+        'members': iplist.get_raw_entries_as_string_list(separator="\n"),
+        'ip4_mapping': ip_map.to_string_list(),
+        'ip4_count': ip_map.count_ips()
+    }
+
+    matched_workloads: List[pylo.Workload] = []
+
+    for workload, wkl_map in workloads_ip4maps_cache.items():
+        affected_rows = ip_map.substract(wkl_map)
+        if affected_rows > 0:
+            print("matched workload   {}".format(workload.get_name()))
+            matched_workloads.append(workload)
+            appgroup_tracker[workload.get_appgroup_str()] = True
+        #print(ip_map.print_to_std(header="after substraction", padding="     "))
+
+
+    new_row['ip4_uncovered_count'] = ip_map.count_ips()
+    new_row['covered_workloads_count'] = len(matched_workloads)
+    new_row['covered_workloads_list'] = pylo.string_list_to_text(matched_workloads, "\n")
+    new_row['covered_workloads_appgroups'] = pylo.string_list_to_text(appgroup_tracker.keys(), "\n")
+
+    csv_report.add_line_from_object(new_row)
+
+
+# <editor-fold desc="Building Workloads ip4 Cache">
+workloads_ip4maps_cache: Dict[pylo.Workload, pylo.IP4Map] = {}
+print(" * Building Workloads IP4 mapping... ", end='')
+for workload in org.WorkloadStore.get_managed_workloads_list():
+    ip_map = workload.get_ip4map_from_interfaces()
+    workloads_ip4maps_cache[workload] = ip_map
+print("OK")
+# </editor-fold>
+
+# <editor-fold desc="Building IPLists ip4 Cache">
+iplists_ip4maps_cache: Dict[pylo.IPList, pylo.IP4Map] = {}
+print(" * Building IPLists IP4 mapping... ", end='')
+for iplist in org.IPListStore.items_by_href.values():
+    ip_map = iplist.get_ip4map()
+    iplists_ip4maps_cache[iplist] = ip_map
+print("OK")
+# </editor-fold>
+
+
+print(" * Now analyzing IPLists:", flush=True)
+for (iplist, ip_map) in iplists_ip4maps_cache.items():
+    add_iplist_to_report(iplist)
+
+print(" ** DONE **")
+
+print()
+print(" * Writing report file '{}' ... ".format(report_file), end='', flush=True)
+csv_report.write_to_csv(report_file)
+print("DONE")
+print(" * Writing report file '{}' ... ".format(report_file_excel), end='', flush=True)
+csv_report.write_to_excel(report_file_excel)
+print("DONE")
+
+if csv_report.lines_count() < 1:
+    print("\n** WARNING: no entry matched your filters so reports are empty !\n")
+
+

illumio_pylo/utilities/iplists_stats_duplicates_unused_finder.py

@@ -0,0 +1,75 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+import illumio_pylo as pylo
+import argparse
+
+
+parser = argparse.ArgumentParser(description='TODO LATER')
+parser.add_argument('--pce', type=str, required=True,
+                    help='hostname of the PCE')
+
+args = vars(parser.parse_args())
+
+hostname = args['pce']
+
+
+org = pylo.Organization(1)
+
+print("Loading PCE configuration from '{}' or cached file... ".format(hostname), end="", flush=True)
+org.load_from_cache_or_saved_credentials(hostname, prompt_for_api_key_if_missing=True)
+print("OK!\n")
+
+unique_hashes = {}
+unique_ipranges = {}
+total_ipranges = 0
+unused_iplists = []
+
+for iplist in org.IPListStore.items_by_href.values():
+    #print("- handling of iplist '{}' with {} members".format(iplist.name, iplist.count_entries()))
+    entries_hash = ""
+    for entry in sorted(iplist.raw_entries):
+        total_ipranges += 1
+
+        entries_hash += entry + ' '
+        if entry not in unique_ipranges:
+            unique_ipranges[entry] = [iplist]
+        else:
+            unique_ipranges[entry].append(iplist)
+
+    #print("  - hash: {}".format(entries_hash))
+
+    if entries_hash in unique_hashes:
+        unique_hashes[entries_hash].append(iplist)
+    else:
+        unique_hashes[entries_hash] = [iplist]
+
+    if iplist.count_references() == 0:
+        unused_iplists.append(iplist)
+
+    #print(pylo.nice_json(iplist.raw_json))
+
+
+print("*** Listing Unused IPLists ***")
+for iplist in unused_iplists:
+    print(" - '{}' HREF:{}".format(iplist.name, iplist.href))
+
+
+print("*** Listing Duplicate IPLists ***")
+for hash in unique_hashes:
+    if len(unique_hashes[hash]) <= 1:
+        continue
+    print(" - hash {}'".format(hash))
+    for iplist in unique_hashes[hash]:
+        print("    - {} HREF:{}".format(iplist.name, iplist.href))
+
+
+print("\nIPlist count: {}\nUnique iplists: {}\nIPRanges count: {}\nUnique IPRanges: {}".format(org.IPListStore.count(),
+                                                                                               len(unique_hashes),
+                                                                                               total_ipranges,
+                                                                                               len(unique_ipranges)
+                                                                                               ))
+print("Unused IPlists: {}".format(len(unused_iplists)))
+
+print("\nGeneric Config statistics:\n{}".format(org.stats_to_str()))

illumio_pylo/utilities/resources/iplists-import-example.csv

@@ -0,0 +1,3 @@
+name,description,networks
+iplist4,test desc,"192.168.1.33,192.167.50.0-192.167.51.0,192.168.57.0/24,10.11.12.13/32"
+iplist8,misc,"192.168.1.33,192.167.50.0-192.167.51.0,192.168.57.0/24,10.11.12.13/32"

illumio_pylo/utilities/resources/workload-exporter-filter-example.csv

@@ -0,0 +1,3 @@
+hostname,random_name
+test,
+VEN-KFMg,

illumio_pylo/utilities/resources/workloads-import-example.csv

@@ -0,0 +1,2 @@
+name,hostname,ip,role,app,env,loc
+VEN-KFMGF,hostdd,192.168.50.12,R_WEB,A_APPC,E_DEV,L_AMER