illumio-pylo 0.3.1__tar.gz → 0.3.3__tar.gz

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/.github/workflows/make-binaries.yml

@@ -39,18 +39,6 @@ jobs:
         run: |
           mv dist/windows/cli.exe dist/windows/pylo-cli.exe
 
-      - name: Publish executables for Master branch with 'latest' tag
-        # only if this is a TAG type
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-
-        uses: "marvinpinto/action-automatic-releases@latest"
-        with:
-          repo_token: "${{ secrets.GITHUB_TOKEN }}"
-          automatic_release_tag: "latest"
-          prerelease: true
-          title: "Latest Binaries"
-          files: |
-            dist/windows/*
 
       - name: Publish executables along with the tag for a release
         # only if this is a TAG type
@@ -64,14 +52,27 @@ jobs:
           draft: false
           prerelease: false
 
+
+      - name: Publish executables for Master branch with 'latest' tag
+        # only if this is a TAG type
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        uses: "crowbarmaster/GH-Automatic-Releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "latest"
+          prerelease: true
+          title: "Latest Binaries"
+          files: |
+            dist/windows/*
+
+
       - name: Publish executables for Dev branch
         if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
-        uses: softprops/action-gh-release@975c1b265e11dd76618af1c374e7981f9a6ff44a  # v0.1.16
+        uses: "crowbarmaster/GH-Automatic-Releases@latest"
         with:
-          files: dist/windows/*
-          token: ${{ secrets.GITHUB_TOKEN }}
-          draft: false
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "dev-latest"
           prerelease: false
-          tag_name: "dev-build"
-          name: "Latest DEV/Unstable Binaries"
-
+          title: "Latest Binaries for DEV/Unstable branch"
+          files: |
+            dist/windows/*

{illumio_pylo-0.3.1/illumio_pylo.egg-info → illumio_pylo-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: illumio_pylo
-Version: 0.3.1
+Version: 0.3.3
 Summary: A set of tools and library for working with Illumio PCE
 Home-page: https://github.com/cpainchaud/pylo
 Author: Christophe Painchaud

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/examples/explorer_query.py

@@ -60,7 +60,7 @@ explorer_query.filters.set_time_from_x_days_ago(5)
 # Query the PCE for traffic logs matching the filter
 print("Querying PCE for traffic logs matching the filter... ", end='', flush=True)
 traffic_logs = explorer_query.execute()
-print("OK! found {} traffic logs".format(traffic_logs.count_results()))
+print("OK! found {} traffic logs".format(traffic_logs.count_records()))
 
 # Print the results
 records = traffic_logs.get_all_records()

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/API/APIConnector.py

@@ -2,7 +2,7 @@ import json
 import time
 import getpass
 
-from .CredentialsManager import is_api_key_encrypted, decrypt_api_key
+from .CredentialsManager import is_api_key_encrypted, decrypt_api_key, CredentialProfile
 from .JsonPayloadTypes import LabelGroupObjectJsonStructure, LabelObjectCreationJsonStructure, \
     LabelObjectJsonStructure, LabelObjectUpdateJsonStructure, PCEObjectsJsonStructure, \
     LabelGroupObjectUpdateJsonStructure, IPListObjectCreationJsonStructure, IPListObjectJsonStructure, \
@@ -12,7 +12,7 @@ from .JsonPayloadTypes import LabelGroupObjectJsonStructure, LabelObjectCreation
     LabelDimensionObjectStructure, AuditLogApiReplyEventJsonStructure, WorkloadsGetQueryLabelFilterJsonStructure, \
     NetworkDeviceObjectJsonStructure, NetworkDeviceEndpointObjectJsonStructure, HrefReference, \
     WorkloadObjectCreateJsonStructure, WorkloadObjectMultiCreateJsonRequestPayload, \
-    WorkloadBulkUpdateEntryJsonStructure, WorkloadBulkUpdateResponseEntry
+    WorkloadBulkUpdateEntryJsonStructure, WorkloadBulkUpdateResponseEntry, VenObjectJsonStructure
 
 try:
     import requests as requests
@@ -108,6 +108,12 @@ class APIConnector:
     def get_all_object_types():
         return all_object_types.copy()
 
+    @staticmethod
+    def create_from_credentials_object(credentials: CredentialProfile) -> Optional['APIConnector']:
+        return APIConnector(credentials.fqdn, credentials.port, credentials.api_user,
+                            credentials.api_key, skip_ssl_cert_check=not credentials.verify_ssl,
+                            org_id=credentials.org_id, name=credentials.name)
+
     @staticmethod
     def create_from_credentials_in_file(fqdn_or_profile_name: str, request_if_missing: bool = False,
                                         credential_file: Optional[str] = None) -> Optional['APIConnector']:
@@ -115,9 +121,7 @@ class APIConnector:
         credentials = pylo.get_credentials_from_file(fqdn_or_profile_name, credential_file)
 
         if credentials is not None:
-            return APIConnector(credentials.fqdn, credentials.port, credentials.api_user,
-                                credentials.api_key, skip_ssl_cert_check=not credentials.verify_ssl,
-                                org_id=credentials.org_id, name=credentials.name)
+            return APIConnector.create_from_credentials_object(credentials)
 
         if not request_if_missing:
             return None
@@ -362,7 +366,7 @@ class APIConnector:
         else:
             raise pylo.PyloEx("Unsupported object type '{}'".format(object_type))
 
-    def get_pce_objects(self, include_deleted_workloads=False, list_of_objects_to_load: Optional[List[str]] = None):
+    def get_pce_objects(self, include_deleted_workloads=False, list_of_objects_to_load: Optional[List[str]] = None, force_async_mode=False):
 
         objects_to_load = {}
         if list_of_objects_to_load is not None:
@@ -389,7 +393,7 @@ class APIConnector:
         errors = []
         thread_queue = Queue()
 
-        def get_objects(q: Queue, thread_num: int):
+        def get_objects(q: Queue, thread_num: int, force_async_mode=False):
             while True:
                 object_type, errors = q.get()
                 try:
@@ -397,7 +401,7 @@ class APIConnector:
                         q.task_done()
                         continue
                     if object_type == 'workloads':
-                        if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
+                        if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls or force_async_mode:
                             data['workloads'] = self.objects_workload_get(include_deleted=include_deleted_workloads)
                         else:
                             data['workloads'] = self.objects_workload_get(include_deleted=include_deleted_workloads, async_mode=False, max_results=default_max_objects_for_sync_calls)
@@ -456,7 +460,7 @@ class APIConnector:
                 q.task_done()
 
         for i in range(threads_count):
-            worker = Thread(target=get_objects, args=(thread_queue, i))
+            worker = Thread(target=get_objects, args=(thread_queue, i, force_async_mode,))
             worker.daemon = True
             worker.start()
 
@@ -620,6 +624,35 @@ class APIConnector:
 
         return None
 
+    def objects_ven_get(self,
+                             include_deleted=False,
+                             filter_by_ip: str = None,
+                             filter_by_label: Optional[WorkloadsGetQueryLabelFilterJsonStructure] = None,
+                             filter_by_name: str = None,
+                             max_results: int = None,
+                             async_mode=True) -> List[VenObjectJsonStructure]:
+        path = '/vens'
+        data = {}
+
+        if include_deleted:
+            data['include_deleted'] = 'yes'
+
+        if filter_by_ip is not None:
+            data['ip_address'] = filter_by_ip
+
+        if filter_by_label is not None:
+            # filter_by_label must be converted to json text
+            data['labels'] = json.dumps(filter_by_label)
+
+        if filter_by_name is not None:
+            data['name'] = filter_by_name
+
+        if max_results is not None:
+            data['max_results'] = max_results
+
+        return self.do_get_call(path=path, async_call=async_mode, params=data)
+
+
     def objects_workload_get(self,
                              include_deleted=False,
                              filter_by_ip: str = None,

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/API/CredentialsManager.py

@@ -14,7 +14,6 @@ except ImportError:
     paramiko = None
 
 
-
 class CredentialFileEntry(TypedDict):
     name: str
     fqdn: str
@@ -97,7 +96,7 @@ def get_all_credentials_from_file(credential_file: str ) -> List[CredentialProfi
 
 
 def get_credentials_from_file(fqdn_or_profile_name: str = None,
-                              credential_file: str = None) -> CredentialProfile:
+                              credential_file: str = None, fail_with_an_exception=True) -> Optional[CredentialProfile]:
 
     if fqdn_or_profile_name is None:
         log.debug("No fqdn_or_profile_name provided, profile_name=default will be used")
@@ -121,9 +120,12 @@ def get_credentials_from_file(fqdn_or_profile_name: str = None,
         if credential_profile.fqdn.lower() == fqdn_or_profile_name.lower():
             return credential_profile
 
-    raise PyloEx("No profile found in credential file '{}' with fqdn: {}".
+    if fail_with_an_exception:
+        raise PyloEx("No profile found in credential file '{}' with fqdn: {}".
                     format(credential_file, fqdn_or_profile_name))
 
+    return None
+
 
 def list_potential_credential_files() -> List[str]:
     """
@@ -204,9 +206,7 @@ def create_credential_in_default_file(data: CredentialFileEntry) -> str:
     return file_path
 
 
-def encrypt_api_key_with_paramiko_key(ssh_key: paramiko.AgentKey, api_key: str) -> str:
-
-
+def encrypt_api_key_with_paramiko_ssh_key_fernet(ssh_key: paramiko.AgentKey, api_key: str) -> str:
     def encrypt(raw: str, key: bytes) -> bytes:
         """
 
@@ -220,7 +220,7 @@ def encrypt_api_key_with_paramiko_key(ssh_key: paramiko.AgentKey, api_key: str)
 
 
     # generate a random 128bit key
-    session_key_to_sign = os.urandom(16)
+    session_key_to_sign = os.urandom(32)
 
     signed_message = ssh_key.sign_ssh_data(session_key_to_sign)
 
@@ -236,7 +236,7 @@ def encrypt_api_key_with_paramiko_key(ssh_key: paramiko.AgentKey, api_key: str)
     return api_key
 
 
-def decrypt_api_key_with_paramiko_key(encrypted_api_key_payload: str) -> str:
+def decrypt_api_key_with_paramiko_ssh_key_fernet(encrypted_api_key_payload: str) -> str:
     def decrypt(token_b64_encoded: str, key: bytes):
         f = Fernet(base64.urlsafe_b64encode(key))
         return f.decrypt(token_b64_encoded).decode('utf-8')
@@ -277,7 +277,7 @@ def decrypt_api_key(encrypted_api_key_payload: str) -> str:
     if not encrypted_api_key_payload.startswith("$encrypted$:"):
         raise PyloEx("Invalid encrypted API key format")
     if encrypted_api_key_payload.startswith("$encrypted$:ssh-Fernet:"):
-        return decrypt_api_key_with_paramiko_key(encrypted_api_key_payload)
+        return decrypt_api_key_with_paramiko_ssh_key_fernet(encrypted_api_key_payload)
 
     raise PyloEx("Unsupported encryption method: {}".format(encrypted_api_key_payload.split(":")[1]))
 

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/API/JsonPayloadTypes.py

@@ -121,6 +121,35 @@ class WorkloadBulkUpdateResponseEntry(TypedDict):
     message: NotRequired[str]
 
 
+class VenObjectWorkloadSummaryJsonStructure(TypedDict):
+    href: str
+    mode: str
+    online: bool
+
+
+class VenObjectJsonStructure(TypedDict):
+    created_at: str
+    created_by: Optional[HrefReferenceWithName]
+    description: Optional[str]
+    hostname: Optional[str]
+    href: str
+    labels: List[HrefReference]
+    name: Optional[str]
+    interfaces: List[WorkloadInterfaceObjectJsonStructure]
+    updated_at: str
+    updated_by: Optional[HrefReferenceWithName]
+    last_heartbeat_at: Optional[str]
+    last_goodbye_at: Optional[str]
+    ven_type: Literal['server', 'endpoint', 'containerized-ven']
+    active_pce_fqdn: Optional[str]
+    target_pce_fqdn: Optional[str]
+    workloads: List[VenObjectWorkloadSummaryJsonStructure]
+    version: Optional[str]
+    os_id: Optional[str]
+    os_version: Optional[str]
+    os_platform: Optional[str]
+    uid: Optional[str]
+
 
 class RuleServiceReferenceObjectJsonStructure(TypedDict):
     href: str

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/Helpers/exports.py

@@ -255,10 +255,9 @@ class ArraysToExcel:
                             new_line.append('=HYPERLINK("{}", "{}")'.format(item,self._headers[item_index].url_text))
                         else:
                             new_line.append(item)
-
-                    length = find_length(new_line[item_index])
-                    if length > columns_max_width[item_index]:
-                        columns_max_width[item_index] = length
+                            length = find_length(new_line[item_index])
+                            if length > columns_max_width[item_index]:
+                                columns_max_width[item_index] = length
 
 
                 xls_data.append(new_line)

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/IPMap.py

@@ -71,6 +71,15 @@ class IP4Map:
             self._entries.append(new_entry)
             self.sort_and_recalculate()
 
+    def add_another_map(self, another_map: 'IP4Map', skip_recalculation=False):
+        for entry in another_map._entries:
+            self._entries.append(entry)
+
+        if skip_recalculation:
+            return
+
+        self.sort_and_recalculate()
+
     def intersection(self, another_map: 'IP4Map'):
 
         inverted_map = IP4Map()

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/LabeledObject.py

@@ -13,7 +13,7 @@ class LabeledObject:
     def get_labels_dict(self):
         return self._labels.copy()
 
-    def get_labels(self):
+    def get_labels(self) -> Iterable['pylo.Label']:
         return self._labels.values()
 
     def set_label(self, label :'pylo.Label'):

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/Organization.py

@@ -135,7 +135,10 @@ class Organization:
             object_to_load = pylo.APIConnector.get_all_object_types()
 
         if self.pce_version is None:
-            raise pylo.PyloEx('Organization has no "version" specified')
+            if 'pce_version' in data:
+                self.pce_version = pylo.SoftwareVersion(data['pce_version'])
+            else:
+                raise pylo.PyloEx('Organization has no "version" specified')
 
         self.LabelStore.load_label_dimensions(data.get('label_dimensions'))
 

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/Rule.py

@@ -4,7 +4,7 @@ from typing import Optional, List, Union, Dict, Any, NewType
 import illumio_pylo as pylo
 from .API.JsonPayloadTypes import RuleServiceReferenceObjectJsonStructure, RuleDirectServiceReferenceObjectJsonStructure
 from illumio_pylo import Workload, Label, LabelGroup, Ruleset, Referencer, SecurityPrincipal, PyloEx, \
-    Service, nice_json, string_list_to_text, find_connector_or_die, VirtualService, IPList
+    Service, nice_json, string_list_to_text, find_connector_or_die, VirtualService, IPList, PortMap
 
 RuleActorsAcceptableTypes = NewType('RuleActorsAcceptableTypes', Union[Workload, Label, LabelGroup, IPList, VirtualService])
 
@@ -250,6 +250,7 @@ class RuleServiceContainer(pylo.Referencer):
         self.owner = owner
         self._items: Dict[Service, Service] = {}
         self._direct_services: List[DirectServiceInRule] = []
+        self._cached_port_map: Optional[PortMap] = None
 
     def load_from_json(self, data_list: List[RuleServiceReferenceObjectJsonStructure|RuleDirectServiceReferenceObjectJsonStructure]):
         ss_store = self.owner.owner.owner.owner.ServiceStore  # make it a local variable for fast lookups
@@ -295,6 +296,8 @@ class RuleServiceContainer(pylo.Referencer):
         :param service:
         :return: True if the service was removed, False if it was not found
         """
+        self._cached_port_map = None
+
         for i in range(0, len(self._direct_services)):
             if self._direct_services[i] is service:
                 del(self._direct_services[i])
@@ -302,6 +305,8 @@ class RuleServiceContainer(pylo.Referencer):
         return False
 
     def add_direct_service(self, service: DirectServiceInRule) -> bool:
+        self._cached_port_map = None
+
         for member in self._direct_services:
             if service is member:
                 return False
@@ -352,6 +357,27 @@ class RuleServiceContainer(pylo.Referencer):
 
         self.owner.raw_json.update(data)
 
+    def get_port_map(self) -> PortMap:
+        """
+        Get a PortMap object with all ports and protocols from all services in this container
+        :return:
+        """
+        if self._cached_port_map is not None:
+            return self._cached_port_map
+
+        result = PortMap()
+        for service in self._items.values():
+            for entry in service.entries:
+                result.add(entry.protocol, entry.port, entry.to_port, skip_recalculation=True)
+        for direct in self._direct_services:
+            result.add(direct.protocol, direct.port, direct.to_port, skip_recalculation=True)
+
+        result.merge_overlapping_maps()
+
+        self._cached_port_map = result
+
+        return result
+
 
 class RuleHostContainer(pylo.Referencer):
     def __init__(self, owner: 'pylo.Rule', name: str):

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/Ruleset.py

@@ -101,29 +101,17 @@ class RulesetScopeEntry:
 
 
     def to_string(self, label_separator = '|', use_href=False):
-        string = 'All' + label_separator
-        if self.app_label is not None:
-            if use_href:
-                string = self.app_label.href + label_separator
-            else:
-                string = self.app_label.name + label_separator
-
-        if self.env_label is None:
-            string += 'All' + label_separator
-        else:
-            if use_href:
-                string += self.env_label.href + label_separator
-            else:
-                string += self.env_label.name + label_separator
-
-        if self.loc_label is None:
-            string += 'All'
-        else:
-            if use_href:
-                string += self.loc_label.href
+        string = ''
+        for label_type in self.owner.owner.owner.owner.LabelStore.label_types:
+            label = self._labels.get(label_type)
+            if len(string) > 0:
+                string += label_separator
+            if label is None:
+                string += 'All'
+            elif use_href:
+                string += label.href
             else:
-                string += self.loc_label.name
-
+                string += label.name
         return string
 
     def is_all_all_all(self):
@@ -156,10 +144,6 @@ class RulesetScopeEntry:
 
 class Ruleset:
 
-    name: str
-    href: Optional[str]
-    description: str
-
     def __init__(self, owner: 'pylo.RulesetStore'):
         self.owner: 'pylo.RulesetStore' = owner
         self.href: Optional[str] = None
@@ -169,6 +153,7 @@ class Ruleset:
         # must keep an ordered list of rules while the dict by href is there for quick searches
         self._rules_by_href: Dict[str, 'pylo.Rule'] = {}
         self._rules: List['pylo.Rule'] = []
+        self.disabled: bool = False
 
     @property
     def rules(self):
@@ -211,6 +196,9 @@ class Ruleset:
             raise pylo.PyloEx("Cannot find Ruleset href in JSON data: \n" + pylo.Helpers.nice_json(data))
         self.href = data['href']
 
+        if 'enabled' in data:
+            self.disabled = not data['enabled']
+
         scopes_json = data.get('scopes')
         if scopes_json is None:
             raise pylo.PyloEx("Cannot find Ruleset scope in JSON data: \n" + pylo.Helpers.nice_json(data))
@@ -287,7 +275,7 @@ class Ruleset:
         if pce_fqdn is None or pce_port is None:
             connector = pylo.find_connector_or_die(self)
             if pce_fqdn is None:
-                pce_fqdn = connector.hostname
+                pce_fqdn = connector.fqdn
             if pce_port is None:
                 pce_port = connector.port
 

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/Service.py

@@ -7,10 +7,20 @@ from typing import *
 
 class PortMap:
     def __init__(self):
-        self._tcp_map = []
-        self._udp_map = []
+        self._tcp_map: List[List[2]] = []  # [start, end]
+        self._udp_map: List[List[2]] = []  # [start, end]
         self._protocol_map = {}
 
+    def copy(self) -> 'PortMap':
+        new_map = PortMap()
+        new_map._tcp_map = self._tcp_map.copy()
+        new_map._udp_map = self._udp_map.copy()
+        new_map._protocol_map = self._protocol_map.copy()
+        return new_map
+
+    def count(self) -> int:
+        return len(self._tcp_map) + len(self._udp_map) + len(self._protocol_map)
+
     def add(self, protocol, start_port: int, end_port: int = None, skip_recalculation=False):
 
         proto = None
@@ -31,67 +41,54 @@ class PortMap:
             return
 
         if start_port is None:
+            start_port = end_port
+
+        if end_port is None:
             end_port = start_port
 
-        new_entry = [start_port, end_port]
+        if proto == 6:
+            self._tcp_map.append([start_port, end_port])
+        else:
+            self._udp_map.append([start_port, end_port])
 
         if not skip_recalculation:
             self.merge_overlapping_maps()
 
-    def merge_overlapping_maps(self):
-        self._sort_maps()
-
-        new_map = []
-
-        cur_entry = None
-
-        for original_entry in self._tcp_map:
-            if cur_entry is None:
-                cur_entry = original_entry
-                continue
-
-            cur_start = cur_entry[0]
-            cur_end = cur_entry[1]
-            new_start = original_entry[0]
-            new_end = original_entry[1]
+    def to_list_of_objects(self) -> List[Dict]:
+        result = []
+        for entry in self._tcp_map:
+            result.append({'proto': 6, 'port': entry[0], 'to_port': entry[1]})
 
-            if new_start > cur_end + 1:
-                new_map.append(cur_entry)
-                continue
+        for entry in self._udp_map:
+            result.append({'proto': 17, 'port': entry[0], 'to_port': entry[1]})
 
-            if new_end > cur_end:
-                cur_entry[1] = new_end
+        for proto in self._protocol_map:
+            result.append({'proto': proto})
 
-        if cur_entry is not None:
-            self._tcp_map = []
-        else:
-            new_map.append(cur_entry)
-            self._tcp_map = new_map
-
-        new_map = []
-
-        for original_entry in self._udp_map:
-            if cur_entry is None:
-                cur_entry = original_entry
-                continue
-
-            cur_start = cur_entry[0]
-            cur_end = cur_entry[1]
-            new_start = original_entry[0]
-            new_end = original_entry[1]
-
-            if new_start > cur_end + 1:
-                new_map.append(cur_entry)
-                continue
+        return result
 
-            if new_end > cur_end:
-                cur_entry[1] = new_end
+    def merge_overlapping_maps(self):
+        self._sort_maps()
 
-        if cur_entry is not None:
-            self._udp_map = []
-        else:
-            new_map.append(cur_entry)
-            self._udp_map = new_map
+        def merge_maps(map_list):
+            new_list = []
+            current = None
+            for entry in map_list:
+                if current is None:
+                    current = entry
+                    continue
+
+                if entry[0] <= current[1] + 1:
+                    current[1] = entry[1]
+                else:
+                    new_list.append(current)
+                    current = entry
+            if current is not None:
+                new_list.append(current)
+            return new_list
+
+        self._tcp_map = merge_maps(self._tcp_map)
+        self._udp_map = merge_maps(self._udp_map)
 
     def _sort_maps(self):
         def first_entry(my_list):

{illumio_pylo-0.3.1 → illumio_pylo-0.3.3}/illumio_pylo/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.1"
+__version__ = "0.3.3"
 
 import os
 import sys