contentctl 4.4.7__py3-none-any.whl → 5.0.0__py3-none-any.whl

contentctl/enrichments/cve_enrichment.py

@@ -1,64 +1,70 @@
 from __future__ import annotations
 from pycvesearch import CVESearch
-import functools
-import os
-import shelve
-import time
-from typing import Annotated, Any, Union, TYPE_CHECKING
-from pydantic import ConfigDict, BaseModel,Field, computed_field
+from typing import Annotated, Union, TYPE_CHECKING
+from pydantic import ConfigDict, BaseModel, Field, computed_field
 from decimal import Decimal
-from requests.exceptions import ReadTimeout
 from contentctl.objects.annotated_types import CVE_TYPE
+
 if TYPE_CHECKING:
     from contentctl.objects.config import validate
 
 
-
-CVESSEARCH_API_URL = 'https://cve.circl.lu'
+CVESSEARCH_API_URL = "https://cve.circl.lu"
 
 
 class CveEnrichmentObj(BaseModel):
     id: CVE_TYPE
-    cvss: Annotated[Decimal, Field(ge=.1, le=10, decimal_places=1)]
+    cvss: Annotated[Decimal, Field(ge=0.1, le=10, decimal_places=1)]
     summary: str
-    
+
     @computed_field
     @property
-    def url(self)->str:
+    def url(self) -> str:
         BASE_NVD_URL = "https://nvd.nist.gov/vuln/detail/"
         return f"{BASE_NVD_URL}{self.id}"
 
 
 class CveEnrichment(BaseModel):
     use_enrichment: bool = True
-    cve_api_obj: Union[CVESearch,None] = None
+    cve_api_obj: Union[CVESearch, None] = None
 
     # Arbitrary_types are allowed to let us use the CVESearch Object
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        frozen=True
-    )
+    model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
     @staticmethod
-    def getCveEnrichment(config:validate, timeout_seconds:int=10, force_disable_enrichment:bool=True)->CveEnrichment:
+    def getCveEnrichment(
+        config: validate,
+        timeout_seconds: int = 10,
+        force_disable_enrichment: bool = True,
+    ) -> CveEnrichment:
         if force_disable_enrichment:
-            return CveEnrichment(use_enrichment=False, cve_api_obj=None)    
-        
+            return CveEnrichment(use_enrichment=False, cve_api_obj=None)
+
         if config.enrichments:
             try:
                 cve_api_obj = CVESearch(CVESSEARCH_API_URL, timeout=timeout_seconds)
                 return CveEnrichment(use_enrichment=True, cve_api_obj=cve_api_obj)
             except Exception as e:
-                raise Exception(f"Error setting CVE_SEARCH API to: {CVESSEARCH_API_URL}: {str(e)}")
-        
-        return CveEnrichment(use_enrichment=False, cve_api_obj=None)
-
+                raise Exception(
+                    f"Error setting CVE_SEARCH API to: {CVESSEARCH_API_URL}: {str(e)}"
+                )
 
-    def enrich_cve(self, cve_id:str, raise_exception_on_failure:bool=True)->CveEnrichmentObj:
+        return CveEnrichment(use_enrichment=False, cve_api_obj=None)
 
+    def enrich_cve(
+        self, cve_id: str, raise_exception_on_failure: bool = True
+    ) -> CveEnrichmentObj:
         if not self.use_enrichment:
-            return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME")
+            return CveEnrichmentObj(
+                id=cve_id,
+                cvss=Decimal(5.0),
+                summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME",
+            )
         else:
             print("WARNING - Dynamic enrichment not supported at this time.")
-            return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME")
-        # Depending on needs, we may add dynamic enrichment functionality back to the tool
+            return CveEnrichmentObj(
+                id=cve_id,
+                cvss=Decimal(5.0),
+                summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME",
+            )
+        # Depending on needs, we may add dynamic enrichment functionality back to the tool

contentctl/enrichments/splunk_app_enrichment.py

@@ -1,11 +1,8 @@
 import requests
 import xmltodict
-import json
 import functools
-import pickle
 import shelve
 import os
-import time
 
 SPLUNKBASE_API_URL = "https://apps.splunk.com/api/apps/entriesbyid/"
 
@@ -13,15 +10,16 @@ APP_ENRICHMENT_CACHE_FILENAME = "lookups/APP_ENRICHMENT_CACHE.db"
 
 NON_PERSISTENT_CACHE = {}
 
+
 @functools.cache
-def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes:
+def requests_get_helper(url: str, force_cached_or_offline: bool = False) -> bytes:
     if force_cached_or_offline:
         if not os.path.exists(APP_ENRICHMENT_CACHE_FILENAME):
             print(f"Cache at {APP_ENRICHMENT_CACHE_FILENAME} not found - Creating it.")
-        cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag='c', writeback=True)
+        cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag="c", writeback=True)
     else:
         cache = NON_PERSISTENT_CACHE
-    
+
     if url in cache:
         req_content = cache[url]
     else:
@@ -29,62 +27,66 @@ def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes:
             req = requests.get(url)
             req_content = req.content
             cache[url] = req_content
-        except Exception as e:
-            raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}"))
-    
+        except Exception:
+            raise (
+                Exception(
+                    f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}"
+                )
+            )
+
     if isinstance(cache, shelve.Shelf):
-        #close the cache if it is a shelf
+        # close the cache if it is a shelf
         cache.close()
-    
+
     return req_content
 
-class SplunkAppEnrichment():
 
+class SplunkAppEnrichment:
     @classmethod
-    def enrich_splunk_app(self, splunk_ta: str, force_cached_or_offline: bool = False) -> dict:
-        
+    def enrich_splunk_app(
+        self, splunk_ta: str, force_cached_or_offline: bool = False
+    ) -> dict:
         appurl = SPLUNKBASE_API_URL + splunk_ta
         splunk_app_enriched = dict()
-        
+
         try:
-            
             content = requests_get_helper(appurl, force_cached_or_offline)
             response_dict = xmltodict.parse(content)
-            
+
             # check if list since data changes depending on answer
             url, results = self._parse_splunkbase_response(response_dict)
             # grab the app name
             for i in results:
-                if i['@name'] == 'appName':
-                    splunk_app_enriched['name'] = i['#text']
-            # grab out the splunkbase url  
-            if 'entriesbyid' in url:
+                if i["@name"] == "appName":
+                    splunk_app_enriched["name"] = i["#text"]
+            # grab out the splunkbase url
+            if "entriesbyid" in url:
                 content = requests_get_helper(url, force_cached_or_offline)
                 response_dict = xmltodict.parse(content)
-                
-                #print(json.dumps(response_dict, indent=2))
+
+                # print(json.dumps(response_dict, indent=2))
                 url, results = self._parse_splunkbase_response(response_dict)
                 # chop the url so we grab the splunkbase portion but not direct download
-                splunk_app_enriched['url'] = url.rsplit('/', 4)[0]
+                splunk_app_enriched["url"] = url.rsplit("/", 4)[0]
         except requests.exceptions.ConnectionError as connErr:
             print(f"There was a connErr for ta {splunk_ta}: {connErr}")
             # there was a connection error lets just capture the name
-            splunk_app_enriched['name'] = splunk_ta
-            splunk_app_enriched['url'] = ''
+            splunk_app_enriched["name"] = splunk_ta
+            splunk_app_enriched["url"] = ""
         except Exception as e:
-            print(f"There was an unknown error enriching the Splunk TA [{splunk_ta}]: {str(e)}")
-            splunk_app_enriched['name'] = splunk_ta
-            splunk_app_enriched['url'] = ''
-
+            print(
+                f"There was an unknown error enriching the Splunk TA [{splunk_ta}]: {str(e)}"
+            )
+            splunk_app_enriched["name"] = splunk_ta
+            splunk_app_enriched["url"] = ""
 
         return splunk_app_enriched
 
     def _parse_splunkbase_response(response_dict):
-        if isinstance(response_dict['feed']['entry'], list):
-            url = response_dict['feed']['entry'][0]['link']['@href']
-            results = response_dict['feed']['entry'][0]['content']['s:dict']['s:key']
+        if isinstance(response_dict["feed"]["entry"], list):
+            url = response_dict["feed"]["entry"][0]["link"]["@href"]
+            results = response_dict["feed"]["entry"][0]["content"]["s:dict"]["s:key"]
         else:
-            url = response_dict['feed']['entry']['link']['@href']
-            results = response_dict['feed']['entry']['content']['s:dict']['s:key']
+            url = response_dict["feed"]["entry"]["link"]["@href"]
+            results = response_dict["feed"]["entry"]["content"]["s:dict"]["s:key"]
         return url, results
-

contentctl/helper/link_validator.py

@@ -1,7 +1,8 @@
 from pydantic import BaseModel, model_validator
 from typing import Union, Callable, Any
 import requests
-import urllib3, urllib3.exceptions
+import urllib3
+import urllib3.exceptions
 import time
 import abc
 
@@ -10,88 +11,96 @@ import shelve
 
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
 ALLOWED_HTTP_CODES = [200]
-class LinkStats(BaseModel):
 
-    #Static Values
+
+class LinkStats(BaseModel):
+    # Static Values
     method: Callable = requests.get
-    allowed_http_codes: list[int] = ALLOWED_HTTP_CODES 
-    access_count: int = 1 #when constructor is called, it has been accessed once!
+    allowed_http_codes: list[int] = ALLOWED_HTTP_CODES
+    access_count: int = 1  # when constructor is called, it has been accessed once!
     timeout_seconds: int = 15
     allow_redirects: bool = True
     headers: dict = {"User-Agent": DEFAULT_USER_AGENT_STRING}
     verify_ssl: bool = False
     if verify_ssl is False:
         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-    
-    #Values generated at runtime.
-    #We need to assign these some default values to get the 
-    #validation working since ComputedField has not yet been 
-    #introduced to Pydantic
+
+    # Values generated at runtime.
+    # We need to assign these some default values to get the
+    # validation working since ComputedField has not yet been
+    # introduced to Pydantic
     reference: str
     referencing_files: set[str]
-    redirect: Union[str,None] = None
+    redirect: Union[str, None] = None
     status_code: int = 0
     valid: bool = False
-    resolution_time: float = 0 
-    
-    
-    def is_link_valid(self, referencing_file:str)->bool:
+    resolution_time: float = 0
+
+    def is_link_valid(self, referencing_file: str) -> bool:
         self.access_count += 1
         self.referencing_files.add(referencing_file)
         return self.valid
-    
+
     @model_validator(mode="before")
-    def check_reference(cls, data:Any)->Any:
+    def check_reference(cls, data: Any) -> Any:
         start_time = time.time()
-        #Get out all the fields names to make them easier to reference
-        method = data['method']
-        reference = data['reference']
-        timeout_seconds = data['timeout_seconds']
-        headers = data['headers']
-        allow_redirects = data['allow_redirects']
-        verify_ssl = data['verify_ssl']
-        allowed_http_codes = data['allowed_http_codes']
+        # Get out all the fields names to make them easier to reference
+        method = data["method"]
+        reference = data["reference"]
+        timeout_seconds = data["timeout_seconds"]
+        headers = data["headers"]
+        allow_redirects = data["allow_redirects"]
+        verify_ssl = data["verify_ssl"]
+        allowed_http_codes = data["allowed_http_codes"]
         if not (reference.startswith("http://") or reference.startswith("https://")):
-            raise(ValueError(f"Reference {reference} does not begin with http(s). Only http(s) references are supported"))
-        
+            raise (
+                ValueError(
+                    f"Reference {reference} does not begin with http(s). Only http(s) references are supported"
+                )
+            )
+
         try:
-            get = method(reference, timeout=timeout_seconds, 
-                            headers = headers, 
-                            allow_redirects=allow_redirects, verify=verify_ssl)
+            get = method(
+                reference,
+                timeout=timeout_seconds,
+                headers=headers,
+                allow_redirects=allow_redirects,
+                verify=verify_ssl,
+            )
             resolution_time = time.time() - start_time
-            data['status_code'] = get.status_code
-            data['resolution_time'] = resolution_time
+            data["status_code"] = get.status_code
+            data["resolution_time"] = resolution_time
             if reference != get.url:
-                data['redirect'] = get.url
+                data["redirect"] = get.url
             else:
-                data['redirect'] = None #None is also already the default
+                data["redirect"] = None  # None is also already the default
 
-            #Returns the updated values and sets them for the object
+            # Returns the updated values and sets them for the object
             if get.status_code in allowed_http_codes:
-                data['valid'] = True
+                data["valid"] = True
             else:
-                #print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
-                data['valid'] = False
-            return data    
+                # print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
+                data["valid"] = False
+            return data
 
-        except Exception as e:
+        except Exception:
             resolution_time = time.time() - start_time
-            #print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
-            data['status_code'] = 0
-            data['valid'] = False
-            data['redirect'] = None
-            data['resolution_time'] = resolution_time
+            # print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
+            data["status_code"] = 0
+            data["valid"] = False
+            data["redirect"] = None
+            data["resolution_time"] = resolution_time
             return data
 
 
 class LinkValidator(abc.ABC):
-    cache: Union[dict[str,LinkStats], shelve.Shelf] = {}
+    cache: Union[dict[str, LinkStats], shelve.Shelf] = {}
     uncached_checks: int = 0
     total_checks: int = 0
-    #cache: dict[str,LinkStats] = {}
+    # cache: dict[str,LinkStats] = {}
 
     use_file_cache: bool = False
-    reference_cache_file: str ="lookups/REFERENCE_CACHE.db"
+    reference_cache_file: str = "lookups/REFERENCE_CACHE.db"
 
     @staticmethod
     def initialize_cache(use_file_cache: bool = False):
@@ -99,74 +108,88 @@ class LinkValidator(abc.ABC):
         if use_file_cache is False:
             return
         if not os.path.exists(LinkValidator.reference_cache_file):
-            print(f"Cache at {LinkValidator.reference_cache_file} not found - Creating it.")
-        
+            print(
+                f"Cache at {LinkValidator.reference_cache_file} not found - Creating it."
+            )
+
         try:
-            LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True)
-        except:
-            print(f"Failed to create the cache file {LinkValidator.reference_cache_file}.  Reference info will not be cached.")
+            LinkValidator.cache = shelve.open(
+                LinkValidator.reference_cache_file, flag="c", writeback=True
+            )
+        except Exception:
+            print(
+                f"Failed to create the cache file {LinkValidator.reference_cache_file}.  Reference info will not be cached."
+            )
             LinkValidator.cache = {}
 
-        #Remove all of the failures to force those resources to be resolved again
+        # Remove all of the failures to force those resources to be resolved again
         failed_refs = []
         for ref in LinkValidator.cache.keys():
             if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES:
                 failed_refs.append(ref)
-                #can't remove it here because this will throw an error:
-                #cannot change size of dictionary while iterating over it
+                # can't remove it here because this will throw an error:
+                # cannot change size of dictionary while iterating over it
             else:
-                #Set the reference count to 0 and referencing files to empty set
+                # Set the reference count to 0 and referencing files to empty set
                 LinkValidator.cache[ref].access_count = 0
                 LinkValidator.cache[ref].referencing_files = set()
-        
-        for ref in failed_refs:
-            del(LinkValidator.cache[ref])
 
+        for ref in failed_refs:
+            del LinkValidator.cache[ref]
 
-        
-             
     @staticmethod
     def close_cache():
         if LinkValidator.use_file_cache:
             LinkValidator.cache.close()
 
     @staticmethod
-    def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool:
+    def validate_reference(
+        reference: str, referencing_file: str, raise_exception_if_failure: bool = False
+    ) -> bool:
         LinkValidator.total_checks += 1
         if reference not in LinkValidator.cache:
             LinkValidator.uncached_checks += 1
-            LinkValidator.cache[reference] = LinkStats(reference=reference, referencing_files = set([referencing_file]))
+            LinkValidator.cache[reference] = LinkStats(
+                reference=reference, referencing_files=set([referencing_file])
+            )
         result = LinkValidator.cache[reference].is_link_valid(referencing_file)
 
-        #print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
+        # print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
 
         if result is True:
             return True
         elif raise_exception_if_failure is True:
-            raise(Exception(f"Reference Link Failed: {reference}"))
+            raise (Exception(f"Reference Link Failed: {reference}"))
         else:
             return False
+
     @staticmethod
     def print_link_validation_errors():
-        failures = [LinkValidator.cache[k] for k in LinkValidator.cache if LinkValidator.cache[k].valid is False]
+        failures = [
+            LinkValidator.cache[k]
+            for k in LinkValidator.cache
+            if LinkValidator.cache[k].valid is False
+        ]
         failures.sort(key=lambda d: d.status_code)
         for failure in failures:
-            print(f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:")
+            print(
+                f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:"
+            )
             for ref in failure.referencing_files:
                 print(f"\t* {ref}")
 
     @staticmethod
-    def SecurityContentObject_validate_references(v:list, values: dict)->list:
-        if 'check_references' not in values:
-            raise(Exception("Member 'check_references' missing from Baseline!"))
-        elif values['check_references'] is False:
-            #Reference checking is enabled
+    def SecurityContentObject_validate_references(v: list, values: dict) -> list:
+        if "check_references" not in values:
+            raise (Exception("Member 'check_references' missing from Baseline!"))
+        elif values["check_references"] is False:
+            # Reference checking is enabled
             pass
-        elif values['check_references'] is True:
+        elif values["check_references"] is True:
             for reference in v:
-                LinkValidator.validate_reference(reference, values['name'])
-                #Remove the check_references key from the values dict so that it is not
-        #output by the serialization code
-        del values['check_references']
+                LinkValidator.validate_reference(reference, values["name"])
+                # Remove the check_references key from the values dict so that it is not
+        # output by the serialization code
+        del values["check_references"]
 
         return v