regscale-cli 6.16.1.0__py3-none-any.whl → 6.16.2.0__py3-none-any.whl

regscale/integrations/public/fedramp/fedramp_common.py

@@ -1878,11 +1878,11 @@ def check_control_list_length(ssp_obj: SSP, mapping: dict) -> Optional[list]:
         return missing_controls
 
 
-def log_controls_info(ssp_obj: dict, current_imps: List[dict]) -> None:
+def log_controls_info(ssp_obj: SSP, current_imps: List[dict]) -> None:
     """
     Log controls info
 
-    :param dict ssp_obj: SSP object
+    :param SSP ssp_obj: SSP object
     :param List[dict] current_imps: List of current implementations
     :return None:
     :rtype None:
@@ -2305,8 +2305,8 @@ def fetch_existing_stakeholders(api: Api, config: dict, regscale_ssp: dict) -> l
     if response.ok:
         logger.info(
             f"Found {len(response.json())} existing stakeholders",
-            record_type="stackholders",
-            model_layer="stackholders",
+            record_type="stakeholders",
+            model_layer="stakeholders",
         )
         return response.json()
     return []

regscale/integrations/public/fedramp/inventory_items.py

@@ -59,7 +59,7 @@ def get_dict_value(outer_key: str, inner_key: str, source_dict: Dict) -> Optiona
     outer_dict = source_dict.get(outer_key, None)
     if outer_dict is not None and isinstance(outer_dict, dict):
         val = outer_dict.get(inner_key, None)
-        logger.info(f"{val}")
+        logger.debug(f"{val=}")
         return val
     return None
 
@@ -205,8 +205,8 @@ def parse_inventory_items(trv: FedrampTraversal, components_dict: Dict):
 
                 for imp_component in implemented_components:
                     if not isinstance(components_dict, dict):
-                        logger.info(components_dict)
-                        logger.info("components do not exist yet")
+                        logger.debug(components_dict)
+                        logger.info("Components do not exist yet.")
                         continue
                     comp = components_dict.get(imp_component.get("component_uuid"))
                     comp_id = comp.get("id") if comp else None

regscale/integrations/scanner_integration.py

@@ -9,12 +9,12 @@ import enum
 import hashlib
 import json
 import logging
-import re
 import threading
 import time
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import Any, Dict, Generic, Iterator, List, Optional, Set, TypeVar, Union
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, Generic, Iterator, List, Optional, Set, TypeVar, Union, Callable
 
 from rich.progress import Progress, TaskID
 
@@ -345,6 +345,15 @@ class IntegrationFinding:
     :param Optional[str] remediation: The remediation of the finding, defaults to None.
     :param Optional[str] source_rule_id: The source rule ID of the finding, defaults to None.
     :param Optional[str] poam_id: The POAM ID of the finding, defaults to None.
+    :param Optional[str] cvss_v3_vector: The CVSS v3 vector of the finding, defaults to None.
+    :param Optional[str] cvss_v2_vector: The CVSS v2 vector of the finding, defaults to None.
+    :param Optional[str] affected_os: The affected OS of the finding, defaults to None.
+    :param Optional[str] image_digest: The image digest of the finding, defaults to None.
+    :param Optional[str] affected_packages: The affected packages of the finding, defaults to None.
+    :param Optional[str] installed_versions: The installed versions of the finding, defaults to None.
+    :param Optional[str] fixed_versions: The fixed versions of the finding, defaults to None.
+    :param Optional[str] fix_status: The fix status of the finding, defaults to None.
+    :param Optional[str] build_version: The build version of the finding, defaults to None.
     """
 
     control_labels: List[str]
@@ -367,6 +376,16 @@ class IntegrationFinding:
     dns: Optional[str] = None
     severity_int: int = 0
     security_check: Optional[str] = None
+    cvss_v3_vector: Optional[str] = None
+    cvss_v2_vector: Optional[str] = None
+    affected_os: Optional[str] = None
+    package_path: Optional[str] = None
+    image_digest: Optional[str] = None
+    affected_packages: Optional[str] = None
+    installed_versions: Optional[str] = None
+    fixed_versions: Optional[str] = None
+    fix_status: Optional[str] = None
+    build_version: Optional[str] = None
 
     # Issues
     issue_title: str = ""
@@ -1211,34 +1230,126 @@ class ScannerIntegration(ABC):
 
     def _process_assets(self, assets: Iterator[IntegrationAsset], loading_assets: TaskID) -> int:
         """
-        Processes the assets using single or multi-threaded approach based on THREAD_MAX_WORKERS.
+        Process assets using single or multi-threaded approach based on THREAD_MAX_WORKERS.
 
-        :param Iterator[IntegrationAsset] assets: The assets to process
-        :param TaskID loading_assets: The task ID for the progress bar
-        :return: The number of assets processed
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param TaskID loading_assets: Task ID for the progress bar
+        :return: Number of assets processed
         :rtype: int
         """
-        assets_processed = 0
-        # prime cache
+        self._prime_asset_cache()
+        process_func = self._create_process_function(loading_assets)
+        max_workers = get_thread_workers_max()
+
+        if max_workers == 1:
+            return self._process_single_threaded(assets, process_func)
+        return self._process_multi_threaded(assets, process_func, max_workers)
+
+    def _prime_asset_cache(self) -> None:
+        """
+        Prime the asset cache by fetching assets for the given plan.
+
+        :rtype: None
+        """
         regscale_models.Asset.get_all_by_parent(
             parent_id=self.plan_id, parent_module=regscale_models.SecurityPlan.get_module_string()
         )
 
-        process_func = lambda my_asset: self._process_single_asset(my_asset, loading_assets)  # noqa: E731
+    def _create_process_function(self, loading_assets: TaskID) -> Callable[[IntegrationAsset], bool]:
+        """
+        Create a function to process a single asset.
+
+        :param TaskID loading_assets: Task ID for the progress bar
+        :return: Function that processes an asset and returns success status
+        :rtype: Callable[[IntegrationAsset], bool]
+        """
+        return lambda asset: self._process_single_asset(asset, loading_assets)
+
+    def _process_single_threaded(
+        self, assets: Iterator[IntegrationAsset], process_func: Callable[[IntegrationAsset], bool]
+    ) -> int:
+        """
+        Process assets sequentially in a single thread.
+
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :return: Number of assets processed
+        :rtype: int
+        """
+        assets_processed = 0
+        for asset in assets:
+            if process_func(asset):
+                assets_processed = self._update_processed_count(assets_processed)
+        return assets_processed
+
+    def _process_multi_threaded(
+        self, assets: Iterator[IntegrationAsset], process_func: Callable[[IntegrationAsset], bool], max_workers: int
+    ) -> int:
+        """
+        Process assets in batches using multiple threads.
+
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :param int max_workers: Maximum number of worker threads
+        :return: Number of assets processed
+        :rtype: int
+        """
+        batch_size = max_workers * 2
+        assets_processed = 0
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            batch = []
+            futures = []
 
-        if get_thread_workers_max() == 1:
             for asset in assets:
-                if process_func(asset):
-                    assets_processed = self._update_processed_count(assets_processed)
-        else:
-            with concurrent.futures.ThreadPoolExecutor(max_workers=get_thread_workers_max()) as executor:
-                future_to_asset = {executor.submit(process_func, asset): asset for asset in assets}
-                for future in concurrent.futures.as_completed(future_to_asset):
-                    if future.result():
-                        assets_processed = self._update_processed_count(assets_processed)
+                batch.append(asset)
+                if len(batch) >= batch_size:
+                    assets_processed += self._submit_and_process_batch(executor, process_func, batch, futures)
+                    batch = []
+                    futures = []
+
+            if batch:  # Process any remaining items
+                assets_processed += self._submit_and_process_batch(executor, process_func, batch, futures)
+
+        return assets_processed
+
+    def _submit_and_process_batch(
+        self,
+        executor: ThreadPoolExecutor,
+        process_func: Callable[[IntegrationAsset], bool],
+        batch: List[IntegrationAsset],
+        futures: List,
+    ) -> int:
+        """
+        Submit a batch of assets for processing and count successful completions.
+
+        :param ThreadPoolExecutor executor: Thread pool executor for parallel processing
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :param List[IntegrationAsset] batch: Batch of assets to process
+        :param List futures: List to store future objects
+        :return: Number of assets processed in this batch
+        :rtype: int
+        """
+        assets_processed = 0
+        for asset in batch:
+            futures.append(executor.submit(process_func, asset))
+
+        for future in concurrent.futures.as_completed(futures):
+            if future.result():
+                assets_processed = self._update_processed_count(assets_processed)
 
         return assets_processed
 
+    def _update_processed_count(self, current_count: int) -> int:
+        """
+        Increment the processed count.
+
+        :param int current_count: Current number of processed items
+        :return: Updated count
+        :rtype: int
+        """
+        return current_count + 1
+
     def _process_single_asset(self, asset: IntegrationAsset, loading_assets: TaskID) -> bool:
         """
         Processes a single asset and handles any exceptions.
@@ -1430,7 +1541,7 @@ class ScannerIntegration(ABC):
         issue.issueOwnerId = self.assessor_id
         issue.securityPlanId = self.plan_id
         issue.identification = "Vulnerability Assessment"
-        issue.dateFirstDetected = finding.date_created
+        issue.dateFirstDetected = finding.first_seen
         issue.dueDate = finding.due_date
         issue.description = description
         issue.sourceReport = finding.source_report or self.title
@@ -1844,9 +1955,8 @@ class ScannerIntegration(ABC):
         scan_history = self.create_scan_history()
         current_vulnerabilities: Dict[int, Set[int]] = defaultdict(set)
         processed_findings_count = 0
-        findings_to_process = self.num_findings_to_process
         loading_findings = self.finding_progress.add_task(
-            f"[#f8b737]Processing {f'{findings_to_process} ' if findings_to_process else ''}finding(s) from {self.title}",
+            f"[#f8b737]Processing {f'{self.num_findings_to_process} ' if self.num_findings_to_process else ''}finding(s) from {self.title}",
             total=self.num_findings_to_process if self.num_findings_to_process else None,
         )
 
@@ -1865,13 +1975,11 @@ class ScannerIntegration(ABC):
                 self.process_finding(finding_to_process, scan_history, current_vulnerabilities)
                 with count_lock:
                     processed_findings_count += 1
-                    if findings_to_process and self.finding_progress.tasks[loading_findings].total != float(
-                        findings_to_process
-                    ):
+                    if self.num_findings_to_process:
                         self.finding_progress.update(
                             loading_findings,
-                            total=findings_to_process,
-                            description=f"[#f8b737]Processing {findings_to_process} findings from {self.title}.",
+                            total=self.num_findings_to_process,
+                            description=f"[#f8b737]Processing {self.num_findings_to_process} findings from {self.title}.",
                         )
                     self.finding_progress.advance(loading_findings, 1)
             except Exception as exc:
@@ -1885,8 +1993,21 @@ class ScannerIntegration(ABC):
             for finding in findings:
                 process_finding_with_progress(finding)
         else:
+            # Process findings in batches to control memory usage
+            batch_size = get_thread_workers_max() * 2  # Set batch size based on thread count
             with concurrent.futures.ThreadPoolExecutor(max_workers=get_thread_workers_max()) as executor:
-                list(executor.map(process_finding_with_progress, findings))
+                batch = []
+                for finding in findings:
+                    batch.append(finding)
+                    if len(batch) >= batch_size:
+                        # Process this batch
+                        list(executor.map(process_finding_with_progress, batch))
+                        # Clear the batch
+                        batch = []
+
+                # Process any remaining items
+                if batch:
+                    list(executor.map(process_finding_with_progress, batch))
 
         # Close outdated issues
         self._results["scan_history"] = scan_history.save()
@@ -2068,6 +2189,9 @@ class ScannerIntegration(ABC):
                 finding.vpr_score if hasattr(finding, "vprScore") else None
             ),  # If this is the VPR score, otherwise use a different field
             cvsSv3BaseScore=finding.cvss_v3_base_score or finding.cvss_v3_score or finding.cvss_score,
+            cvsSv2BaseScore=finding.cvss_v2_score,
+            cvsSv3BaseVector=finding.cvss_v3_vector,
+            cvsSv2BaseVector=finding.cvss_v2_vector,
             scanId=scan_history.id,
             severity=self.issue_to_vulnerability_map.get(finding.severity, regscale_models.VulnerabilitySeverity.Low),
             description=finding.description,
@@ -2086,23 +2210,30 @@ class ScannerIntegration(ABC):
             port=finding.port if hasattr(finding, "port") else None,
             protocol=finding.protocol if hasattr(finding, "protocol") else None,
             operatingSystem=asset.operating_system if hasattr(asset, "operating_system") else None,
+            fixedVersions=finding.fixed_versions,
+            buildVersion=finding.build_version,
+            fixStatus=finding.fix_status,
+            installedVersions=finding.installed_versions,
+            affectedOS=finding.affected_os,
+            packagePath=finding.package_path,
+            imageDigest=finding.image_digest,
+            affectedPackages=finding.affected_packages,
         )
 
         vulnerability = vulnerability.create_or_update()
-        if re.match(r"^\d+\.\d+(\.\d+){0,2}$", self.regscale_version) or self.regscale_version >= "5.64.0":
-            regscale_models.VulnerabilityMapping(
-                vulnerabilityId=vulnerability.id,
-                assetId=asset.id,
-                scanId=scan_history.id,
-                securityPlansId=self.plan_id,
-                createdById=self.assessor_id,
-                tenantsId=self.tenant_id,
-                isPublic=True,
-                dateCreated=get_current_datetime(),
-                firstSeen=finding.first_seen,
-                lastSeen=finding.last_seen,
-                status=finding.status,
-            ).create_unique()
+        regscale_models.VulnerabilityMapping(
+            vulnerabilityId=vulnerability.id,
+            assetId=asset.id,
+            scanId=scan_history.id,
+            securityPlansId=self.plan_id,
+            createdById=self.assessor_id,
+            tenantsId=self.tenant_id,
+            isPublic=True,
+            dateCreated=get_current_datetime(),
+            firstSeen=finding.first_seen,
+            lastSeen=finding.last_seen,
+            status=finding.status,
+        ).create_unique()
         return vulnerability
 
     def handle_vulnerability(

regscale/models/integration_models/cisa_kev_data.json

@@ -1,9 +1,24 @@
 {
     "title": "CISA Catalog of Known Exploited Vulnerabilities",
-    "catalogVersion": "2025.03.19",
-    "dateReleased": "2025-03-19T18:00:10.5417Z",
-    "count": 1307,
+    "catalogVersion": "2025.03.24",
+    "dateReleased": "2025-03-24T18:01:34.066Z",
+    "count": 1308,
     "vulnerabilities": [
+        {
+            "cveID": "CVE-2025-30154",
+            "vendorProject": "reviewdog",
+            "product": "action-setup GitHub Action",
+            "vulnerabilityName": "reviewdog\/action-setup GitHub Action Embedded Malicious Code Vulnerability",
+            "dateAdded": "2025-03-24",
+            "shortDescription": "reviewdog action-setup GitHub Action contains an embedded malicious code vulnerability that dumps exposed secrets to Github Actions Workflow Logs.",
+            "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.",
+            "dueDate": "2025-04-14",
+            "knownRansomwareCampaignUse": "Unknown",
+            "notes": "This vulnerability affects a common open-source project, third-party library, or a protocol used by different products. For more information, please see: https:\/\/github.com\/reviewdog\/reviewdog\/security\/advisories\/GHSA-qmg3-hpqr-gqvc ; https:\/\/nvd.nist.gov\/vuln\/detail\/CVE-2025-30154",
+            "cwes": [
+                "CWE-506"
+            ]
+        },
         {
             "cveID": "CVE-2017-12637",
             "vendorProject": "SAP",
@@ -55,11 +70,11 @@
             "product": "changed-files GitHub Action",
             "vulnerabilityName": "tj-actions\/changed-files GitHub Action Embedded Malicious Code Vulnerability",
             "dateAdded": "2025-03-18",
-            "shortDescription": "The tj-actions\/changed-files GitHub Action contains an embedded malicious code vulnerability that allows a remote attacker to discover secrets by reading actions logs. These secrets may include, but are not limited to, valid AWS access keys, GitHub personal access tokens (PATs), npm tokens, and private RSA keys.",
+            "shortDescription": "tj-actions\/changed-files GitHub Action contains an embedded malicious code vulnerability that allows a remote attacker to discover secrets by reading Github Actions Workflow Logs. These secrets may include, but are not limited to, valid AWS access keys, GitHub personal access tokens (PATs), npm tokens, and private RSA keys.",
             "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.",
             "dueDate": "2025-04-08",
             "knownRansomwareCampaignUse": "Unknown",
-            "notes": "https:\/\/github.com\/tj-actions\/changed-files\/blob\/45fb12d7a8bedb4da42342e52fe054c6c2c3fd73\/README.md?plain=1#L20-L28 ; https:\/\/nvd.nist.gov\/vuln\/detail\/CVE-2025-30066",
+            "notes": "This vulnerability affects a common open-source project, third-party library, or a protocol used by different products. For more information, please see: https:\/\/github.com\/tj-actions\/changed-files\/blob\/45fb12d7a8bedb4da42342e52fe054c6c2c3fd73\/README.md?plain=1#L20-L28 ; https:\/\/nvd.nist.gov\/vuln\/detail\/CVE-2025-30066",
             "cwes": [
                 "CWE-506"
             ]