regscale-cli 6.16.0.0__py3-none-any.whl → 6.16.1.0__py3-none-any.whl

regscale/__init__.py

@@ -1 +1 @@
-__version__ = "6.16.0.0"
+__version__ = "6.16.1.0"

regscale/core/app/application.py

@@ -86,6 +86,7 @@ class Application(metaclass=Singleton):
             "dependabotRepo": "<myGithubRepoNameGoesHere>",
             "dependabotToken": "<myGithubPersonalAccessTokenGoesHere>",
             "domain": "https://regscale.yourcompany.com/",
+            "disableCache": False,
             "evidenceFolder": "./evidence",
             "passScore": 80,
             "failScore": 30,

regscale/core/app/utils/app_utils.py

@@ -642,7 +642,7 @@ def save_to_json(file: Path, data: Any, output_log: bool) -> None:
         with open(file, "w", encoding="utf-8") as outfile:
             outfile.write(str(data))
     if output_log:
-        logger.info("Data successfully saved to %s", file.name)
+        logger.info("Data successfully saved to %s", file.absolute())
 
 
 def save_data_to(file: Path, data: Any, output_log: bool = True, transpose_data: bool = True) -> None:

regscale/core/app/utils/parser_utils.py

@@ -23,7 +23,7 @@ def safe_float(value: Any, default: float = 0.0, field_name: str = "value") -> f
     try:
         return float(value)
     except (ValueError, TypeError):
-        logger.warning(f"Invalid float {field_name}: {value}. Defaulting to {default}")
+        logger.debug(f"Invalid float {field_name}: {value}. Defaulting to {default}")
         return default
 
 
@@ -43,7 +43,7 @@ def safe_int(value: Any, default: int = 0, field_name: str = "value") -> int:
     try:
         return int(value)
     except (ValueError, TypeError):
-        logger.warning(f"Invalid integer {field_name}: {value}. Defaulting to {default}")
+        logger.debug(f"Invalid integer {field_name}: {value}. Defaulting to {default}")
         return default
 
 

regscale/integrations/commercial/azure/intune.py

@@ -73,6 +73,7 @@ def sync_intune(regscale_id: int, regscale_module: str):
         )
         in_scan.sync_findings(
             plan_id=regscale_id,
+            finding_count=len(devices),
             integration_findings=fetch_intune_findings(devices=devices),
         )
     else:

regscale/integrations/commercial/nessus/scanner.py

@@ -79,6 +79,7 @@ class NessusIntegration(ScannerIntegration):
             self.log_file_warning_and_exit(path)
         if not self.check_collection(file_collection, path):
             return
+        self.num_findings_to_process = 0
         for file in iterate_files(file_collection):
             content = read_file(file)
             root = ET.fromstring(content)
@@ -87,6 +88,7 @@ class NessusIntegration(ScannerIntegration):
                 for nessus_vulnerability in root.iterfind(f"./Report/ReportHost[@name='{asset_name}']/ReportItem"):
                     parsed_vulnerability = self.parse_finding(nessus_vulnerability, asset_name)
                     if parsed_vulnerability:
+                        self.num_findings_to_process += 1
                         yield parsed_vulnerability
         self.move_files(file_collection)
 
@@ -214,6 +216,7 @@ class NessusIntegration(ScannerIntegration):
                 tree = ElementTree(root)
                 assets = nfr.scan.report_hosts(root)
                 cpe_items = cpe_xml_to_dict(tree)  # type: ignore
+                self.num_assets_to_process = len(assets)
                 for asset in assets:
                     asset_properties = self.get_asset_properties(root, cpe_items, asset)
                     parsed_asset = self.parse_asset(asset_properties)

regscale/integrations/commercial/sap/sysdig/sysdig_scanner.py

@@ -74,9 +74,11 @@ class SAPConcurSysDigScanner(ScannerIntegration):
             raise ValueError("Path is required")
 
         logger.info(f"Fetching assets from {path}")
+        self.num_assets_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield self.parse_assets(row)
 
     def fetch_findings(self, *args: Tuple, **kwargs: dict) -> Iterator[IntegrationFinding]:
@@ -94,9 +96,11 @@ class SAPConcurSysDigScanner(ScannerIntegration):
 
         logger.info(f"Fetching findings from {path}")
 
+        self.num_findings_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_findings_to_process += 1
                 yield from self.parse_findings(finding=row, kwargs=kwargs)
 
     def parse_findings(self, finding: Dict[str, Any], **kwargs: dict) -> Iterator[IntegrationFinding]:

regscale/integrations/commercial/sap/tenable/click.py

@@ -16,7 +16,7 @@ logger = logging.getLogger("regscale")
 @regscale_ssp_id()
 @click.option(
     "--path",
-    type=click.STRING,
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
     help="Path to the CSV file containing the SAP Concur data.",
     required=True,
 )

regscale/integrations/commercial/sap/tenable/scanner.py

@@ -30,7 +30,8 @@ class SAPConcurScanner(ScannerIntegration):
         "low": regscale_models.IssueSeverity.Low,
     }
 
-    def parse_assets(self, asset: Dict[str, Any]) -> IntegrationAsset:
+    @staticmethod
+    def parse_assets(asset: Dict[str, Any]) -> IntegrationAsset:
         """
         Parse a single asset from the vulnerability data.
 
@@ -88,9 +89,11 @@ class SAPConcurScanner(ScannerIntegration):
             raise ValueError("Path is required")
 
         logger.info(f"Fetching assets from {path}")
+        self.num_assets_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield self.parse_assets(row)
 
     def fetch_findings(self, *args: Tuple, **kwargs: dict) -> Iterator[IntegrationFinding]:
@@ -108,9 +111,11 @@ class SAPConcurScanner(ScannerIntegration):
 
         logger.info(f"Fetching findings from {path}")
 
+        self.num_findings_to_process = 0
         with open(path, "r", newline="") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
+                self.num_assets_to_process += 1
                 yield from self.parse_findings(row)
 
     def parse_findings(self, finding: Dict[str, Any]) -> Iterator[IntegrationFinding]:
@@ -132,8 +137,9 @@ class SAPConcurScanner(ScannerIntegration):
             for cve in cves:
                 yield self._create_finding(finding, severity, cve.strip())
 
+    @staticmethod
     def _create_finding(
-        self, finding: Dict[str, Any], severity: regscale_models.IssueSeverity, cve: str = ""
+        finding: Dict[str, Any], severity: regscale_models.IssueSeverity, cve: str = ""
     ) -> IntegrationFinding:
         """
         Create an IntegrationFinding object from the given data.

regscale/integrations/commercial/tenablev2/click.py

@@ -3,6 +3,7 @@
 """Tenable integration for RegScale CLI"""
 
 import queue
+from concurrent.futures import wait
 from typing import TYPE_CHECKING, Any
 
 from regscale.integrations.integration_override import IntegrationOverride
@@ -723,16 +724,20 @@ def fetch_vulns(query_id: int = 0, regscale_ssp_id: int = 0):
         # make sure folder exists
         with tempfile.TemporaryDirectory() as temp_dir:
             logger.info("Saving Tenable SC data to disk...%s", temp_dir)
-            consume_iterator_to_file(iterator=vulns, dir_path=Path(temp_dir), scanner=sc)
+            num_assets_processed, num_findings_to_process = consume_iterator_to_file(
+                iterator=vulns, dir_path=Path(temp_dir), scanner=sc
+            )
             iterables = tenable_dir_to_tuple_generator(Path(temp_dir))
             try:
                 sc.sync_assets(
                     plan_id=regscale_ssp_id,
                     integration_assets=(asset for sublist in iterables[0] for asset in sublist),
+                    asset_count=num_assets_processed,
                 )
                 sc.sync_findings(
                     plan_id=regscale_ssp_id,
                     integration_findings=(finding for sublist in iterables[1] for finding in sublist),
+                    finding_count=num_findings_to_process,
                 )
             except IndexError as ex:
                 logger.error("Error processing Tenable SC data: %s", ex)
@@ -754,23 +759,26 @@ def tenable_dir_to_tuple_generator(dir_path: Path):
     return assets_gen, findings_gen
 
 
-def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path, scanner: SCIntegration) -> int:
+def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path, scanner: SCIntegration) -> tuple:
     """
     Consume an iterator and write the results to a file
 
     :param AnalysisResultsIterator iterator: Tenable SC iterator
     :param Path dir_path: The directory to save the pickled files
     :param SCIntegration scanner: Tenable SC Integration object
-    :rtype: int
-    :return: The total count of items processed
+    :rtype: tuple
+    :return: The total count of assets and findings processed
     """
     app = Application()
     logger.info("Consuming Tenable SC iterator...")
     override = IntegrationOverride(app)
+    asset_count = 0
+    findings_count = 0
     total_count = ThreadSafeCounter()
     page_number = ThreadSafeCounter()
     rec_count = ThreadSafeCounter()
     process_list = queue.Queue()
+    futures_lst = []
     with ThreadPoolExecutor(max_workers=5) as executor:
         for dat in iterator:
             total_count.increment()
@@ -778,19 +786,32 @@ def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path,
             rec_count.increment()
             if rec_count.value == len(iterator.page):
                 page_number.increment()
-                executor.submit(
-                    process_sc_chunk,
-                    app=app,
-                    vulns=pop_queue(queue=process_list, queue_len=len(iterator.page)),
-                    page=page_number.value,
-                    dir_path=dir_path,
-                    sc=scanner,
-                    override=override,
+                futures_lst.append(
+                    executor.submit(
+                        process_sc_chunk,
+                        app=app,
+                        vulns=pop_queue(queue=process_list, queue_len=len(iterator.page)),
+                        page=page_number.value,
+                        dir_path=dir_path,
+                        sc=scanner,
+                        override=override,
+                    )
                 )
                 rec_count.set(0)
+    # Collect results from all threads
+    asset_count = 0
+    findings_count = 0
+    # Wait for completion
+    wait(futures_lst)
+
+    for future in futures_lst:
+        findings, assets = future.result()
+        asset_count += assets
+        findings_count += findings
+
     if total_count.value == 0:
         logger.warning("No Tenable SC data found.")
-    return total_count.value
+    return asset_count, findings_count
 
 
 def pop_queue(queue: queue.Queue, queue_len: int) -> list:
@@ -816,12 +837,13 @@ def pop_queue(queue: queue.Queue, queue_len: int) -> list:
     return retrieved_items
 
 
-def process_sc_chunk(**kwargs) -> None:
+def process_sc_chunk(**kwargs) -> tuple:
     """
     Process Tenable SC chunk
 
     :param kwargs: Keyword arguments
-    :rtype: None
+    :rtype: tuple
+    :return: Tuple of findings and assets
     """
     # iterator.page, iterator.page_count, file_path, query_id, ssp_id
     integration_mapping = kwargs.get("override")
@@ -833,7 +855,7 @@ def process_sc_chunk(**kwargs) -> None:
     tenable_sc: SCIntegration = kwargs.get("sc")
     thread = current_thread()
     if not len(vulns):
-        return
+        return (0, 0)
     # I can't add a to-do thanks to sonarlint, but we need to add CVE lookup from plugin id
     # append file to path
     # Process to RegScale
@@ -857,6 +879,7 @@ def process_sc_chunk(**kwargs) -> None:
         kwargs.get("page"),
     )
     logger.debug(f"Completed thread: name={thread.name}, idnet={get_ident()}, id={get_native_id()}")
+    return (len(findings), len(assets))
 
 
 def get_last_pull_epoch(regscale_ssp_id: int) -> int:

regscale/integrations/commercial/wizv2/click.py

@@ -394,13 +394,6 @@ def add_report_evidence(
     default="NIST800-53R5",
     required=True,
 )
-@click.option(  # type: ignore
-    "--include_not_implemented",
-    "-n",
-    is_flag=True,
-    help="Include not implemented controls",
-    default=False,
-)
 def sync_compliance(
     wiz_project_id,
     regscale_id,
@@ -409,21 +402,16 @@ def sync_compliance(
     client_secret,
     catalog_id,
     framework,
-    include_not_implemented,
 ):
     """Sync compliance posture from Wiz to RegScale"""
-    from regscale.core.app.utils.app_utils import create_progress_object
     from regscale.integrations.commercial.wizv2.utils import _sync_compliance
 
-    compliance_job_progress = create_progress_object()
-    with compliance_job_progress:
-        _sync_compliance(
-            wiz_project_id=wiz_project_id,
-            regscale_id=regscale_id,
-            regscale_module=regscale_module,
-            include_not_implemented=include_not_implemented,
-            client_id=client_id,
-            client_secret=client_secret,
-            catalog_id=catalog_id,
-            framework=framework,
-        )
+    _sync_compliance(
+        wiz_project_id=wiz_project_id,
+        regscale_id=regscale_id,
+        regscale_module=regscale_module,
+        client_id=client_id,
+        client_secret=client_secret,
+        catalog_id=catalog_id,
+        framework=framework,
+    )

regscale/integrations/commercial/wizv2/scanner.py

@@ -101,6 +101,7 @@ class WizVulnerabilityIntegration(ScannerIntegration):
             raise ValueError("Wiz project ID is required")
 
         logger.info("Fetching Wiz findings...")
+        self.num_findings_to_process = 0
 
         for wiz_vulnerability_type in self.get_query_types(project_id=project_id):
             logger.info("Fetching Wiz findings for %s...", wiz_vulnerability_type["type"])
@@ -114,8 +115,8 @@ class WizVulnerabilityIntegration(ScannerIntegration):
                 topic_key=wiz_vulnerability_type["topic_key"],
                 file_path=wiz_vulnerability_type["file_path"],
             )
+            self.num_findings_to_process += len(nodes)
             yield from self.parse_findings(nodes, wiz_vulnerability_type["type"])
-
         logger.info("Finished fetching Wiz findings.")
 
     def parse_findings(