prancer-basic 3.0.26__tar.gz → 3.0.28__tar.gz

{prancer-basic-3.0.26/src/prancer_basic.egg-info → prancer-basic-3.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: prancer-basic
-Version: 3.0.26
+Version: 3.0.28
 Summary: Prancer Basic, http://prancer.io/
 Home-page: https://github.com/prancer-io/cloud-validation-framework
 Author: Farshid M/Ajey Khanapuri

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/setup.py

@@ -18,7 +18,7 @@ LONG_DESCRIPTION = """
 setup(
     name='prancer-basic',
     # also update the version in processor.__init__.py file
-    version='3.0.26',
+    version='3.0.28',
     description='Prancer Basic, http://prancer.io/',
     long_description=LONG_DESCRIPTION,
     license = "BSD",

{prancer-basic-3.0.26 → prancer-basic-3.0.28/src/prancer_basic.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: prancer-basic
-Version: 3.0.26
+Version: 3.0.28
 Summary: Prancer Basic, http://prancer.io/
 Home-page: https://github.com/prancer-io/cloud-validation-framework
 Author: Farshid M/Ajey Khanapuri

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/prancer_basic.egg-info/SOURCES.txt

@@ -58,6 +58,7 @@ src/processor/connector/special_compliance/compliances.py
 src/processor/connector/special_crawler/__init__.py
 src/processor/connector/special_crawler/azure_crawler.py
 src/processor/connector/special_crawler/base_crawler.py
+src/processor/connector/special_crawler/google_crawler.py
 src/processor/connector/special_node_pull/__init__.py
 src/processor/connector/special_node_pull/azure_node_pull.py
 src/processor/connector/special_node_pull/base_node_pull.py
@@ -98,6 +99,8 @@ src/processor/helper/utils/jinjatemplates/fs_connector.json
 src/processor/helper/utils/jinjatemplates/git_connector.json
 src/processor/helper/utils/jinjatemplates/mastersnapshot.json
 src/processor/helper/utils/jinjatemplates/mastertest.json
+src/processor/helper/xml/__init__.py
+src/processor/helper/xml/xml_utils.py
 src/processor/helper/yaml/__init__.py
 src/processor/helper/yaml/yaml_utils.py
 src/processor/logging/__init__.py

prancer-basic-3.0.28/src/processor/__init__.py

@@ -0,0 +1,3 @@
+# Prancer Basic
+
+__version__ = '3.0.28'

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/connector/snapshot_azure.py

@@ -133,7 +133,7 @@ def get_all_nodes(token, sub_name, sub_id, node, user, snapshot_source):
                 put_in_currentdata('errors', data)
                 logger.info("Get Id returned invalid status: %s", status)
         
-        azure_crawler = AzureCrawler(resources, token=token, apiversions=get_api_versions(), subscription_id=sub_id, version=version, scope=scope)
+        azure_crawler = AzureCrawler(resources=resources, token=token, apiversions=get_api_versions(), subscription_id=sub_id, version=version, scope=scope)
         resources = azure_crawler.check_for_special_crawl(nodetype)
         if resources:
             for idx, value in enumerate(resources):

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/connector/snapshot_google.py

@@ -20,6 +20,7 @@ import os
 import re
 from googleapiclient import discovery
 from oauth2client.service_account import ServiceAccountCredentials
+from processor.connector.special_crawler.google_crawler import GoogleCrawler
 from processor.helper.file.file_utils import exists_file
 from processor.logging.log_handler import getlogger
 from processor.helper.httpapi.http_utils import http_get_request
@@ -191,6 +192,8 @@ def get_params_for_get_method(response, url_var, project_id):
             elif item == r"{secret}":
                 secret_name = response['name']
                 params[item] = secret_name.split('/')[-1]
+            elif item == r"{selfLink}" and response.get("selfLink"):
+                params[item] = response['selfLink']
 
         return params
     except Exception as ex:
@@ -198,6 +201,9 @@ def get_params_for_get_method(response, url_var, project_id):
 
 def get_request_url_get_method(get_method, item, project_id=None):
     request_url, _ = get_method_api_path(get_method)
+    if not request_url:
+        return request_url
+
     url_list = request_url.split('/')
     url_var = []
     for elements in url_list:
@@ -207,6 +213,7 @@ def get_request_url_get_method(get_method, item, project_id=None):
 
     params = get_params_for_get_method(item , url_var, project_id)
     request_url = requested_get_method_url(request_url, params)
+    request_url = re.sub(r'(?<!:)\/\/', '/', request_url)
     return request_url
 
 def get_request_url_list_method(get_method, list_method, item, project_id=None, credentials=None):
@@ -273,6 +280,9 @@ def get_node(credentials, node, snapshot_source, snapshot):
             "Authorization" : ("Bearer %s" % access_token)
         }
 
+        status = False
+        data = {}
+
         if get_method:
             node_type = node['get_method'] if node and 'get_method' in node else ""
             if isinstance(node_type, list) and len(node_type) > 1:
@@ -286,7 +296,14 @@ def get_node(credentials, node, snapshot_source, snapshot):
             else:
                 logger.error("Invalid node type %s", node_type)
                 return db_record
-            if method == "POST":
+
+            google_crawler = GoogleCrawler(access_token=access_token, project_id=project_id)
+            process_status, data = google_crawler.check_for_data_process(path, node_type)
+
+            if process_status:
+                status = 200
+
+            elif method == "POST":
                 base_url = "%s%s" % (base_node_type, ".googleapis.com")
                 request_url = "https://%s/%s" % (base_url, path)
                 logger.info("Invoke request for get snapshot: %s", request_url)
@@ -391,22 +408,26 @@ def get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data):
             "Authorization" : ("Bearer %s" % access_token)
         }
 
-        base_node_type_list = node_type.split("/")
-        if len(base_node_type_list) > 1:
-            base_node_type = "/".join(base_node_type_list[1:])
-        else:
-            logger.error("Invalid node type '%s'", node_type)
-            return db_record
+        google_crawler = GoogleCrawler(access_token=access_token, project_id=project_id)
+        crawler_status, data = google_crawler.check_for_special_crawl(node_type)
 
-        request_url = get_api_path(base_node_type)
-        if not request_url:
-            logger.error("API URL not set in google parameters for resource type: %s", base_node_type)
+        if not crawler_status:
+            base_node_type_list = node_type.split("/")
+            if len(base_node_type_list) > 1:
+                base_node_type = "/".join(base_node_type_list[1:])
+            else:
+                logger.error("Invalid node type '%s'", node_type)
+                return db_record
+
+            request_url = get_api_path(base_node_type)
+            if not request_url:
+                logger.error("API URL not set in google parameters for resource type: %s", base_node_type)
 
-        request_url = generate_request_url(request_url, project_id)
-        logger.info("Invoke request for get snapshot: %s", request_url)
+            request_url = generate_request_url(request_url, project_id)
+            logger.info("Invoke request for get snapshot: %s", request_url)
         
-        status, data = http_get_request(request_url, header)
-        logger.info('Get snapshot status: %s', status)
+            status, data = http_get_request(request_url, header)
+            logger.info('Get snapshot status: %s', status)
 
         fn_str_list = ""
         if node and 'type' in node and node['type']:

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/connector/special_crawler/azure_crawler.py

@@ -6,7 +6,9 @@ from processor.helper.httpapi.restapi_azure import get_access_token, GRAPH_TOKEN
 class AzureCrawler(BaseCrawler):
 
     def __init__(self, resources, **kwargs):
-        super().__init__(resources, **kwargs)
+        super().__init__(**kwargs)
+
+        self.resources = resources
         self.token = kwargs.get("token")
         self.apiversions = kwargs.get("apiversions")
         self.subscription_id = kwargs.get("subscription_id")

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/connector/special_crawler/base_crawler.py

@@ -1,7 +1,6 @@
 
 class BaseCrawler:
-    def __init__(self, resources, **kwargs):
-        self.resources = resources
+    def __init__(self, **kwargs):
         self.resource_types = {}
 
     def check_for_special_crawl(self, resource_type):

prancer-basic-3.0.28/src/processor/connector/special_crawler/google_crawler.py

@@ -0,0 +1,173 @@
+from processor.logging.log_handler import getlogger
+from processor.connector.special_crawler.base_crawler import BaseCrawler
+from processor.helper.xml.xml_utils import xml_to_json
+import requests
+import zipfile
+import io
+import re
+
+logger = getlogger()
+
+class GoogleCrawler(BaseCrawler):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.response_data = None
+        self.status = False
+        self.project_id = kwargs.get("project_id")
+        self.access_token = kwargs.get("access_token")
+        self.special_resource_types = {
+            "apigee/organizations.apis.deployments.list" : self.crawl_apigee_deployments,
+        }
+        self.data_processing_resource_types = {
+            "apigee/organizations.apis.revisions.get" : self.process_apigee_version_data,
+        }
+
+    def get_header(self):
+        return {
+            "Authorization" : ("Bearer %s" % self.access_token)
+        }
+
+    def check_for_special_crawl(self, resource_type):
+        """ 
+        check the resource type need special crawling or not. if it need special crawling then 
+        it will process the special crawler and returns the updated resources
+        """
+        if resource_type in self.special_resource_types:
+            self.status = True
+            crawl_function = self.special_resource_types.get(resource_type)
+            self.response_data = crawl_function()
+
+        return self.status, self.response_data
+    
+    def check_for_data_process(self, path, resource_type):
+        """ 
+        check the resource type need processing on the data. if it need processing then 
+        it will process it and returns the updated resource data
+        """
+        self.path = path
+        if resource_type in self.data_processing_resource_types:
+            self.status = True
+            process_data_function = self.data_processing_resource_types.get(resource_type)
+            self.response_data = process_data_function()
+
+        return self.status, self.response_data
+
+
+    def process_apigee_version_data(self):
+        response_data = {}
+        match = re.search(r"organizations/([^/]+)/apis/([^/]+)/revisions/([^/]+)", self.path)
+        if match:
+            organization = match.group(1)
+            api = match.group(2)
+            revision = match.group(3)
+        else:
+            logger.error("Invalid path for process apigee version")
+            return
+
+        request_url = f"https://apigee.googleapis.com/{self.path}?format=bundle"
+        response = requests.get(url=request_url, headers=self.get_header())
+
+        pattern_dict = {
+            "policies": r'.+/policies/[^/]+\.xml',
+            "proxies": r'.+/proxies/[^/]+\.xml',
+            "targets": r'.+/targets/[^/]+\.xml'
+        }
+
+        if response.status_code == 200:
+            bundle_zip = io.BytesIO(response.content)
+            policies_json = []
+            proxies_json = []
+            targets_json = []
+            with zipfile.ZipFile(bundle_zip, 'r') as zip_ref:                
+                for file in zip_ref.filelist:
+                    for pattern_key, pattern in pattern_dict.items():
+                        matches = re.findall(pattern, file.filename)
+                        if matches:
+                            with zip_ref.open(file.filename) as xml_file:
+                                xml_content = xml_file.read()
+                                json_content = xml_to_json(xml_content)
+
+                                if pattern_key == "policies":
+                                    policies_json.append(json_content)
+                                elif pattern_key == "proxies":
+                                    proxies_json.append(json_content)
+                                elif pattern_key == "targets":
+                                    targets_json.append(json_content)
+                                
+                        
+            response_data = {
+                "organization" : organization,
+                "api" : api,
+                "revision" : revision,
+                "policies" : policies_json,
+                "proxies" : proxies_json,
+                "targets" : targets_json
+            }
+            return response_data
+        else:
+            logger.error(f"Failed to retrieve API bundle. Status code: {response.status_code}, Error: {response.content}")
+            return response_data
+
+    def get_apigee_organizations(self):
+        organizations = []
+        request_url = "https://apigee.googleapis.com/v1/organizations"
+        response = requests.get(url=request_url, headers=self.get_header())
+        if response.status_code != 200:
+            logger.error(f"Failed to get the organization list. Status code: {response.status_code}, Error: {response.content}")
+            return organizations
+
+        data = response.json()
+        organizations = data.get("organizations", [])
+        return organizations
+    
+    def get_apigee_apis(self, organization):
+        apis = []
+        request_url = f"https://apigee.googleapis.com/v1/organizations/{organization}/apis"
+        response = requests.get(url=request_url, headers=self.get_header())
+        if response.status_code != 200:
+            logger.error(f"Failed to get the apigee apis. Status code: {response.status_code}, Error: {response.content}")
+            return apis
+
+        data = response.json()
+        apis = data.get("proxies", [])
+        return apis
+    
+    def get_apigee_deployments(self, organization, api):
+        deployments = []
+        request_url = f"https://apigee.googleapis.com/v1/organizations/{organization}/apis/{api}/deployments"
+        response = requests.get(url=request_url, headers=self.get_header())
+        if response.status_code != 200:
+            logger.error(f"Failed to get the apigee deployments. Status code: {response.status_code}, Error: {response.content}")
+            return deployments
+
+        data = response.json()
+        deployments = data.get("deployments", [])
+        return deployments
+
+    def crawl_apigee_deployments(self):
+        """
+        crawl "apigee/organizations.apis.deployments.list" resource type
+        """
+        deployment_list = []
+        organizations = self.get_apigee_organizations()        
+        for organization in organizations:
+            if organization.get("projectId") != self.project_id:
+                continue
+            organization_name = organization.get("organization")
+            apigee_apis = self.get_apigee_apis(organization_name)            
+            for api in apigee_apis:
+                api_name = api.get("name")
+                deployments = self.get_apigee_deployments(organization_name, api_name)
+                for deployment in deployments:
+                    revision = deployment.get("revision")
+                    deployment["selfLink"] = f"/organizations/{organization_name}/apis/{api_name}/revisions/{revision}"
+                    deployment_list.append(deployment)
+        response_data = {
+            "items" : deployment_list
+        }
+        return response_data
+                    
+
+        

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/crawler/utils.py

@@ -166,6 +166,8 @@ def populate_gcp_projects(container, source, structure_data):
             private_key = user.get("private_key")
             name = re.sub(r"-\d+$", "", name)
             account_list += get_projects_list(private_key_id, private_key, client_email, client_id, name)
+        else:
+            account_list = structure_data.get("projects")
     updated_data = copy.deepcopy(structure_data)
     updated_data["projects"] = account_list
     update_collection_data(container, source, updated_data, dbname, collection)
@@ -199,6 +201,7 @@ def get_projects_list(private_key_id, private_key, client_email, client_id, test
     """ Get google projects list """
     project_list = []
     isremote = get_from_currentdata('remote')
+    private_key_set = True if private_key else False
     if not private_key and isremote:
         private_key = get_value_from_customer_keyvault(private_key_id)
         logger.info('Private key from customer keyvault, Secret: %s', '*' * len(private_key))
@@ -233,7 +236,7 @@ def get_projects_list(private_key_id, private_key, client_email, client_id, test
                                     "client_id": client_id,
                                     "name": "%s-%s" % (test_user, index),
                                     "private_key_id": private_key_id,
-                                    "private_key" : private_key if private_key else "",
+                                    "private_key" : private_key if private_key_set else "",
                                     "type": "service_account"
                                 }
                             ]

prancer-basic-3.0.28/src/processor/helper/xml/xml_utils.py

@@ -0,0 +1,19 @@
+import xml.etree.ElementTree as ET
+
+def parse_element(element):
+    parsed = {
+        "name": element.tag,
+        "text": element.text.strip() if element.text and element.text.strip() else None,
+        "attributes": element.attrib,
+        "children": []
+    }
+
+    for child in element:
+        parsed["children"].append(parse_element(child))
+
+    return parsed
+
+def xml_to_json(xml_content):
+    root = ET.fromstring(xml_content)
+    parsed_xml = parse_element(root)
+    return parsed_xml

{prancer-basic-3.0.26 → prancer-basic-3.0.28}/src/processor/logging/log_handler.py

@@ -196,6 +196,11 @@ class MongoDBHandler(logging.Handler):
     """
     def __init__(self, dburl, dbname, collection='logs', log_type=None):
         logging.Handler.__init__(self)
+
+        self.max_docsize = 15.5*1024*1024
+        self.cursize = 0
+        self.index = 0
+
         try:
             dbconnection =  MongoClient(host=dburl, serverSelectionTimeoutMS=3000)
             _ = dbconnection.list_database_names()
@@ -220,7 +225,7 @@ class MongoDBHandler(logging.Handler):
         self.collection = self.db[self.coll_name]
         DBLOGGER = get_dblog_name(log_type)
         self.dblog_name = DBLOGGER
-        self.collection.insert_one({'name': self.dblog_name, 'logs': []})
+        self.collection.insert_one({'name': self.dblog_name, 'index': self.index, 'logs': []})
 
     def get_log_collection(self):
         return self.dblog_name
@@ -242,10 +247,20 @@ class MongoDBHandler(logging.Handler):
             "log_type": getattr(record, "type", "DEFAULT")
         }
 
+        try:
+            self.cursize += len(json.dumps(db_record))
+        except:
+            self.cursize += len(str(db_record))
+        
+        if self.cursize // self.max_docsize >= 1:
+            self.index += 1
+            self.cursize = 0
+            self.collection.insert_one({'name': self.dblog_name, 'index': self.index, 'logs': []})
+
         try:
             if self.collection is not None and self.dblog_name:
                 # self.collection.insert(db_record, check_keys=False)
-                self.collection.update_one({'name': self.dblog_name}, {'$push': {'logs': db_record}})
+                self.collection.update_one({'name': self.dblog_name, 'index': self.index}, {'$push': {'logs': db_record}})
         except Exception as e:
             print('CRITICAL Logger DB ERROR: Logging to database not possible!')
 

prancer-basic-3.0.26/src/processor/__init__.py

@@ -1,3 +0,0 @@
-# Prancer Basic
-
-__version__ = '3.0.26'