contentctl 5.0.0a2__py3-none-any.whl → 5.0.1__py3-none-any.whl

contentctl/helper/link_validator.py

@@ -1,7 +1,8 @@
 from pydantic import BaseModel, model_validator
 from typing import Union, Callable, Any
 import requests
-import urllib3, urllib3.exceptions
+import urllib3
+import urllib3.exceptions
 import time
 import abc
 
@@ -10,88 +11,96 @@ import shelve
 
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
 ALLOWED_HTTP_CODES = [200]
-class LinkStats(BaseModel):
 
-    #Static Values
+
+class LinkStats(BaseModel):
+    # Static Values
     method: Callable = requests.get
-    allowed_http_codes: list[int] = ALLOWED_HTTP_CODES 
-    access_count: int = 1 #when constructor is called, it has been accessed once!
+    allowed_http_codes: list[int] = ALLOWED_HTTP_CODES
+    access_count: int = 1  # when constructor is called, it has been accessed once!
     timeout_seconds: int = 15
     allow_redirects: bool = True
     headers: dict = {"User-Agent": DEFAULT_USER_AGENT_STRING}
     verify_ssl: bool = False
     if verify_ssl is False:
         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-    
-    #Values generated at runtime.
-    #We need to assign these some default values to get the 
-    #validation working since ComputedField has not yet been 
-    #introduced to Pydantic
+
+    # Values generated at runtime.
+    # We need to assign these some default values to get the
+    # validation working since ComputedField has not yet been
+    # introduced to Pydantic
     reference: str
     referencing_files: set[str]
-    redirect: Union[str,None] = None
+    redirect: Union[str, None] = None
     status_code: int = 0
     valid: bool = False
-    resolution_time: float = 0 
-    
-    
-    def is_link_valid(self, referencing_file:str)->bool:
+    resolution_time: float = 0
+
+    def is_link_valid(self, referencing_file: str) -> bool:
         self.access_count += 1
         self.referencing_files.add(referencing_file)
         return self.valid
-    
+
     @model_validator(mode="before")
-    def check_reference(cls, data:Any)->Any:
+    def check_reference(cls, data: Any) -> Any:
         start_time = time.time()
-        #Get out all the fields names to make them easier to reference
-        method = data['method']
-        reference = data['reference']
-        timeout_seconds = data['timeout_seconds']
-        headers = data['headers']
-        allow_redirects = data['allow_redirects']
-        verify_ssl = data['verify_ssl']
-        allowed_http_codes = data['allowed_http_codes']
+        # Get out all the fields names to make them easier to reference
+        method = data["method"]
+        reference = data["reference"]
+        timeout_seconds = data["timeout_seconds"]
+        headers = data["headers"]
+        allow_redirects = data["allow_redirects"]
+        verify_ssl = data["verify_ssl"]
+        allowed_http_codes = data["allowed_http_codes"]
         if not (reference.startswith("http://") or reference.startswith("https://")):
-            raise(ValueError(f"Reference {reference} does not begin with http(s). Only http(s) references are supported"))
-        
+            raise (
+                ValueError(
+                    f"Reference {reference} does not begin with http(s). Only http(s) references are supported"
+                )
+            )
+
         try:
-            get = method(reference, timeout=timeout_seconds, 
-                            headers = headers, 
-                            allow_redirects=allow_redirects, verify=verify_ssl)
+            get = method(
+                reference,
+                timeout=timeout_seconds,
+                headers=headers,
+                allow_redirects=allow_redirects,
+                verify=verify_ssl,
+            )
             resolution_time = time.time() - start_time
-            data['status_code'] = get.status_code
-            data['resolution_time'] = resolution_time
+            data["status_code"] = get.status_code
+            data["resolution_time"] = resolution_time
             if reference != get.url:
-                data['redirect'] = get.url
+                data["redirect"] = get.url
             else:
-                data['redirect'] = None #None is also already the default
+                data["redirect"] = None  # None is also already the default
 
-            #Returns the updated values and sets them for the object
+            # Returns the updated values and sets them for the object
             if get.status_code in allowed_http_codes:
-                data['valid'] = True
+                data["valid"] = True
             else:
-                #print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
-                data['valid'] = False
-            return data    
+                # print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
+                data["valid"] = False
+            return data
 
-        except Exception as e:
+        except Exception:
             resolution_time = time.time() - start_time
-            #print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
-            data['status_code'] = 0
-            data['valid'] = False
-            data['redirect'] = None
-            data['resolution_time'] = resolution_time
+            # print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
+            data["status_code"] = 0
+            data["valid"] = False
+            data["redirect"] = None
+            data["resolution_time"] = resolution_time
             return data
 
 
 class LinkValidator(abc.ABC):
-    cache: Union[dict[str,LinkStats], shelve.Shelf] = {}
+    cache: Union[dict[str, LinkStats], shelve.Shelf] = {}
     uncached_checks: int = 0
     total_checks: int = 0
-    #cache: dict[str,LinkStats] = {}
+    # cache: dict[str,LinkStats] = {}
 
     use_file_cache: bool = False
-    reference_cache_file: str ="lookups/REFERENCE_CACHE.db"
+    reference_cache_file: str = "lookups/REFERENCE_CACHE.db"
 
     @staticmethod
     def initialize_cache(use_file_cache: bool = False):
@@ -99,74 +108,88 @@ class LinkValidator(abc.ABC):
         if use_file_cache is False:
             return
         if not os.path.exists(LinkValidator.reference_cache_file):
-            print(f"Cache at {LinkValidator.reference_cache_file} not found - Creating it.")
-        
+            print(
+                f"Cache at {LinkValidator.reference_cache_file} not found - Creating it."
+            )
+
         try:
-            LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True)
-        except:
-            print(f"Failed to create the cache file {LinkValidator.reference_cache_file}.  Reference info will not be cached.")
+            LinkValidator.cache = shelve.open(
+                LinkValidator.reference_cache_file, flag="c", writeback=True
+            )
+        except Exception:
+            print(
+                f"Failed to create the cache file {LinkValidator.reference_cache_file}.  Reference info will not be cached."
+            )
             LinkValidator.cache = {}
 
-        #Remove all of the failures to force those resources to be resolved again
+        # Remove all of the failures to force those resources to be resolved again
         failed_refs = []
         for ref in LinkValidator.cache.keys():
             if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES:
                 failed_refs.append(ref)
-                #can't remove it here because this will throw an error:
-                #cannot change size of dictionary while iterating over it
+                # can't remove it here because this will throw an error:
+                # cannot change size of dictionary while iterating over it
             else:
-                #Set the reference count to 0 and referencing files to empty set
+                # Set the reference count to 0 and referencing files to empty set
                 LinkValidator.cache[ref].access_count = 0
                 LinkValidator.cache[ref].referencing_files = set()
-        
-        for ref in failed_refs:
-            del(LinkValidator.cache[ref])
 
+        for ref in failed_refs:
+            del LinkValidator.cache[ref]
 
-        
-             
     @staticmethod
     def close_cache():
         if LinkValidator.use_file_cache:
             LinkValidator.cache.close()
 
     @staticmethod
-    def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool:
+    def validate_reference(
+        reference: str, referencing_file: str, raise_exception_if_failure: bool = False
+    ) -> bool:
         LinkValidator.total_checks += 1
         if reference not in LinkValidator.cache:
             LinkValidator.uncached_checks += 1
-            LinkValidator.cache[reference] = LinkStats(reference=reference, referencing_files = set([referencing_file]))
+            LinkValidator.cache[reference] = LinkStats(
+                reference=reference, referencing_files=set([referencing_file])
+            )
         result = LinkValidator.cache[reference].is_link_valid(referencing_file)
 
-        #print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
+        # print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
 
         if result is True:
             return True
         elif raise_exception_if_failure is True:
-            raise(Exception(f"Reference Link Failed: {reference}"))
+            raise (Exception(f"Reference Link Failed: {reference}"))
         else:
             return False
+
     @staticmethod
     def print_link_validation_errors():
-        failures = [LinkValidator.cache[k] for k in LinkValidator.cache if LinkValidator.cache[k].valid is False]
+        failures = [
+            LinkValidator.cache[k]
+            for k in LinkValidator.cache
+            if LinkValidator.cache[k].valid is False
+        ]
         failures.sort(key=lambda d: d.status_code)
         for failure in failures:
-            print(f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:")
+            print(
+                f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:"
+            )
             for ref in failure.referencing_files:
                 print(f"\t* {ref}")
 
     @staticmethod
-    def SecurityContentObject_validate_references(v:list, values: dict)->list:
-        if 'check_references' not in values:
-            raise(Exception("Member 'check_references' missing from Baseline!"))
-        elif values['check_references'] is False:
-            #Reference checking is enabled
+    def SecurityContentObject_validate_references(v: list, values: dict) -> list:
+        if "check_references" not in values:
+            raise (Exception("Member 'check_references' missing from Baseline!"))
+        elif values["check_references"] is False:
+            # Reference checking is enabled
             pass
-        elif values['check_references'] is True:
+        elif values["check_references"] is True:
             for reference in v:
-                LinkValidator.validate_reference(reference, values['name'])
-                #Remove the check_references key from the values dict so that it is not
-        #output by the serialization code
-        del values['check_references']
+                LinkValidator.validate_reference(reference, values["name"])
+                # Remove the check_references key from the values dict so that it is not
+        # output by the serialization code
+        del values["check_references"]
 
         return v

contentctl/helper/splunk_app.py

@@ -39,6 +39,7 @@ class RetryConstant:
 
 class SplunkBaseError(requests.HTTPError):
     """An error raise in communicating with Splunkbase"""
+
     pass
 
 
@@ -50,6 +51,7 @@ class SplunkApp:
 
     class InitializationError(Exception):
         """An initialization error during SplunkApp setup"""
+
         pass
 
     @staticmethod
@@ -68,16 +70,16 @@ class SplunkApp:
             status_forcelist=status_forcelist,
         )
         adapter = HTTPAdapter(max_retries=retry)
-        session.mount('http://', adapter)
-        session.mount('https://', adapter)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
         return session
 
     def __init__(
-            self,
-            app_uid: Optional[int] = None,
-            app_name_id: Optional[str] = None,
-            manual_setup: bool = False,
-            ) -> None:
+        self,
+        app_uid: Optional[int] = None,
+        app_name_id: Optional[str] = None,
+        manual_setup: bool = False,
+    ) -> None:
         if app_uid is None and app_name_id is None:
             raise SplunkApp.InitializationError(
                 "Either app_uid (the numeric app UID e.g. 742) or app_name_id (the app name "
@@ -123,18 +125,22 @@ class SplunkApp:
         if self._app_info_cache is not None:
             return self._app_info_cache
         elif self.app_uid is None:
-            raise SplunkApp.InitializationError("app_uid must be set in order to fetch app info")
+            raise SplunkApp.InitializationError(
+                "app_uid must be set in order to fetch app info"
+            )
 
         # NOTE: auth not required
         # Get app info by uid
         try:
             response = self.requests_retry_session().get(
                 APIEndPoint.SPLUNK_BASE_APP_INFO.format(app_uid=self.app_uid),
-                timeout=RetryConstant.RETRY_INTERVAL
+                timeout=RetryConstant.RETRY_INTERVAL,
             )
             response.raise_for_status()
         except requests.exceptions.RequestException as e:
-            raise SplunkBaseError(f"Error fetching app info for app_uid {self.app_uid}: {str(e)}")
+            raise SplunkBaseError(
+                f"Error fetching app info for app_uid {self.app_uid}: {str(e)}"
+            )
 
         # parse JSON and set cache
         self._app_info_cache: dict = json.loads(response.content)
@@ -156,7 +162,9 @@ class SplunkApp:
         if "appid" in app_info:
             self.app_name_id = app_info["appid"]
         else:
-            raise SplunkBaseError(f"Invalid response from Splunkbase; missing key 'appid': {app_info}")
+            raise SplunkBaseError(
+                f"Invalid response from Splunkbase; missing key 'appid': {app_info}"
+            )
 
     def set_app_uid(self) -> None:
         """
@@ -166,19 +174,25 @@ class SplunkApp:
         if self.app_uid is not None:
             return
         elif self.app_name_id is None:
-            raise SplunkApp.InitializationError("app_name_id must be set in order to fetch app_uid")
+            raise SplunkApp.InitializationError(
+                "app_name_id must be set in order to fetch app_uid"
+            )
 
         # NOTE: auth not required
         # Get app_uid by app_name_id via a redirect
         try:
             response = self.requests_retry_session().get(
-                APIEndPoint.SPLUNK_BASE_GET_UID_REDIRECT.format(app_name_id=self.app_name_id),
+                APIEndPoint.SPLUNK_BASE_GET_UID_REDIRECT.format(
+                    app_name_id=self.app_name_id
+                ),
                 allow_redirects=False,
-                timeout=RetryConstant.RETRY_INTERVAL
+                timeout=RetryConstant.RETRY_INTERVAL,
             )
             response.raise_for_status()
         except requests.exceptions.RequestException as e:
-            raise SplunkBaseError(f"Error fetching app_uid for app_name_id '{self.app_name_id}': {str(e)}")
+            raise SplunkBaseError(
+                f"Error fetching app_uid for app_name_id '{self.app_name_id}': {str(e)}"
+            )
 
         # Extract the app_uid from the redirect path
         if "Location" in response.headers:
@@ -199,7 +213,9 @@ class SplunkApp:
         if "title" in app_info:
             self.app_title = app_info["title"]
         else:
-            raise SplunkBaseError(f"Invalid response from Splunkbase; missing key 'title': {app_info}")
+            raise SplunkBaseError(
+                f"Invalid response from Splunkbase; missing key 'title': {app_info}"
+            )
 
     def __fetch_url_latest_version_info(self) -> str:
         """
@@ -209,12 +225,16 @@ class SplunkApp:
         # retrieve app entries using the app_name_id
         try:
             response = self.requests_retry_session().get(
-                APIEndPoint.SPLUNK_BASE_FETCH_APP_BY_ENTRY_ID.format(app_name_id=self.app_name_id),
-                timeout=RetryConstant.RETRY_INTERVAL
+                APIEndPoint.SPLUNK_BASE_FETCH_APP_BY_ENTRY_ID.format(
+                    app_name_id=self.app_name_id
+                ),
+                timeout=RetryConstant.RETRY_INTERVAL,
             )
             response.raise_for_status()
         except requests.exceptions.RequestException as e:
-            raise SplunkBaseError(f"Error fetching app entries for app_name_id '{self.app_name_id}': {str(e)}")
+            raise SplunkBaseError(
+                f"Error fetching app entries for app_name_id '{self.app_name_id}': {str(e)}"
+            )
 
         # parse xml
         app_xml = xmltodict.parse(response.content)
@@ -231,7 +251,9 @@ class SplunkApp:
                     return entry.get("link").get("@href")
 
         # raise if no entry was found
-        raise SplunkBaseError(f"No app entry found with 'islatest' tag set to True: {self.app_name_id}")
+        raise SplunkBaseError(
+            f"No app entry found with 'islatest' tag set to True: {self.app_name_id}"
+        )
 
     def __fetch_url_latest_version_download(self, info_url: str) -> str:
         """
@@ -241,10 +263,14 @@ class SplunkApp:
         """
         # fetch download info
         try:
-            response = self.requests_retry_session().get(info_url, timeout=RetryConstant.RETRY_INTERVAL)
+            response = self.requests_retry_session().get(
+                info_url, timeout=RetryConstant.RETRY_INTERVAL
+            )
             response.raise_for_status()
         except requests.exceptions.RequestException as e:
-            raise SplunkBaseError(f"Error fetching download info for app_name_id '{self.app_name_id}': {str(e)}")
+            raise SplunkBaseError(
+                f"Error fetching download info for app_name_id '{self.app_name_id}': {str(e)}"
+            )
 
         # parse XML and extract download URL
         build_xml = xmltodict.parse(response.content)
@@ -254,14 +280,18 @@ class SplunkApp:
     def set_latest_version_info(self) -> None:
         # raise if app_name_id not set
         if self.app_name_id is None:
-            raise SplunkApp.InitializationError("app_name_id must be set in order to fetch latest version info")
+            raise SplunkApp.InitializationError(
+                "app_name_id must be set in order to fetch latest version info"
+            )
 
         # fetch the info URL
         info_url = self.__fetch_url_latest_version_info()
 
         # parse out the version number and fetch the download URL
         self.latest_version = info_url.split("/")[-1]
-        self.latest_version_download_url = self.__fetch_url_latest_version_download(info_url)
+        self.latest_version_download_url = self.__fetch_url_latest_version_download(
+            info_url
+        )
 
     def __get_splunk_base_session_token(self, username: str, password: str) -> str:
         """
@@ -309,12 +339,12 @@ class SplunkApp:
         return token_value
 
     def download(
-            self,
-            out: Path,
-            username: str,
-            password: str,
-            is_dir: bool = False,
-            overwrite: bool = False
+        self,
+        out: Path,
+        username: str,
+        password: str,
+        is_dir: bool = False,
+        overwrite: bool = False,
     ) -> Path:
         """
         Given an output path, download the app to the specified location
@@ -336,11 +366,7 @@ class SplunkApp:
         # Get the Splunkbase session token
         token = self.__get_splunk_base_session_token(username, password)
         response = requests.request(
-            "GET",
-            self.latest_version_download_url,
-            cookies={
-                "sessionid": token
-            }
+            "GET", self.latest_version_download_url, cookies={"sessionid": token}
         )
 
         # If the provided output path was a directory we need to try and pull the filename from the
@@ -348,17 +374,21 @@ class SplunkApp:
         if is_dir:
             try:
                 # Pull 'Content-Disposition' from the headers
-                content_disposition: str = response.headers['Content-Disposition']
+                content_disposition: str = response.headers["Content-Disposition"]
 
                 # Attempt to parse the filename as a KV
                 key, value = content_disposition.strip().split("=")
                 if key != "attachment;filename":
-                    raise ValueError(f"Unexpected key in 'Content-Disposition' KV pair: {key}")
+                    raise ValueError(
+                        f"Unexpected key in 'Content-Disposition' KV pair: {key}"
+                    )
 
                 # Validate the filename is the expected .tgz file
                 filename = Path(value.strip().strip('"'))
                 if filename.suffixes != [".tgz"]:
-                    raise ValueError(f"Filename has unexpected extension(s): {filename.suffixes}")
+                    raise ValueError(
+                        f"Filename has unexpected extension(s): {filename.suffixes}"
+                    )
                 out = Path(out, filename)
             except KeyError as e:
                 raise KeyError(
@@ -371,9 +401,7 @@ class SplunkApp:
 
         # Ensure the output path is not already occupied
         if out.exists() and not overwrite:
-            msg = (
-                f"File already exists at {out}, cannot download the app."
-            )
+            msg = f"File already exists at {out}, cannot download the app."
             raise Exception(msg)
 
         # Make any parent directories as needed