regscale-cli 6.16.0.0__py3-none-any.whl → 6.16.2.0__py3-none-any.whl

regscale/integrations/scanner_integration.py

@@ -9,12 +9,12 @@ import enum
 import hashlib
 import json
 import logging
-import re
 import threading
 import time
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import Any, Dict, Generic, Iterator, List, Optional, Set, TypeVar, Union
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, Generic, Iterator, List, Optional, Set, TypeVar, Union, Callable
 
 from rich.progress import Progress, TaskID
 
@@ -345,6 +345,15 @@ class IntegrationFinding:
     :param Optional[str] remediation: The remediation of the finding, defaults to None.
     :param Optional[str] source_rule_id: The source rule ID of the finding, defaults to None.
     :param Optional[str] poam_id: The POAM ID of the finding, defaults to None.
+    :param Optional[str] cvss_v3_vector: The CVSS v3 vector of the finding, defaults to None.
+    :param Optional[str] cvss_v2_vector: The CVSS v2 vector of the finding, defaults to None.
+    :param Optional[str] affected_os: The affected OS of the finding, defaults to None.
+    :param Optional[str] image_digest: The image digest of the finding, defaults to None.
+    :param Optional[str] affected_packages: The affected packages of the finding, defaults to None.
+    :param Optional[str] installed_versions: The installed versions of the finding, defaults to None.
+    :param Optional[str] fixed_versions: The fixed versions of the finding, defaults to None.
+    :param Optional[str] fix_status: The fix status of the finding, defaults to None.
+    :param Optional[str] build_version: The build version of the finding, defaults to None.
     """
 
     control_labels: List[str]
@@ -367,6 +376,16 @@ class IntegrationFinding:
     dns: Optional[str] = None
     severity_int: int = 0
     security_check: Optional[str] = None
+    cvss_v3_vector: Optional[str] = None
+    cvss_v2_vector: Optional[str] = None
+    affected_os: Optional[str] = None
+    package_path: Optional[str] = None
+    image_digest: Optional[str] = None
+    affected_packages: Optional[str] = None
+    installed_versions: Optional[str] = None
+    fixed_versions: Optional[str] = None
+    fix_status: Optional[str] = None
+    build_version: Optional[str] = None
 
     # Issues
     issue_title: str = ""
@@ -537,6 +556,7 @@ class ScannerIntegration(ABC):
     _lock_registry: ThreadSafeDict = ThreadSafeDict()
     _global_lock = threading.Lock()  # Class-level lock
     _kev_data = ThreadSafeDict()  # Class-level lock
+    _results = ThreadSafeDict()
 
     # Error handling
     errors: List[str] = []
@@ -578,7 +598,7 @@ class ScannerIntegration(ABC):
         self.app = Application()
         self.alerted_assets: Set[str] = set()
         self.regscale_version: str = APIHandler().regscale_version  # noqa
-        logger.info(f"RegScale Version: {self.regscale_version}")
+        logger.debug(f"RegScale Version: {self.regscale_version}")
         self.plan_id: int = plan_id
         self.tenant_id: int = tenant_id
         self.components: ThreadSafeList[Any] = ThreadSafeList()
@@ -801,13 +821,20 @@ class ScannerIntegration(ABC):
         :return: The finding identifier
         :rtype: str
         """
+        # We could have a string truncation error platform side on IntegrationFindingId nvarchar(450)
         prefix = f"{self.plan_id}:"
-        if ScannerVariables.tenableGroupByPlugin and finding.plugin_id:
-            return f"{prefix}{finding.plugin_id}"
+        if (
+            ScannerVariables.tenableGroupByPlugin
+            and finding.plugin_id
+            and "tenable" in (finding.source_report or self.title).lower()
+        ):
+            res = f"{prefix}{finding.plugin_id}"
+            return res[:450]
         prefix += finding.cve or finding.plugin_id or finding.rule_id or self.hash_string(finding.external_id).__str__()
         if ScannerVariables.issueCreation.lower() == "perasset":
-            return f"{prefix}:{finding.asset_identifier}"
-        return prefix
+            res = f"{prefix}:{finding.asset_identifier}"
+            return res[:450]
+        return prefix[:450]
 
     def get_or_create_assessment(self, control_implementation_id: int) -> regscale_models.Assessment:
         """
@@ -963,6 +990,9 @@ class ScannerIntegration(ABC):
 
         created, existing_or_new_asset = self.create_new_asset(asset, component=None)
 
+        # update results expects a dict[str, list] to update result counts
+        self.update_result_counts("assets", {"created": [1] if created else [], "updated": [] if created else [1]})
+
         # If the asset is associated with a component, create a mapping between them.
         if existing_or_new_asset and component:
             _was_created, _asset_mapping = regscale_models.AssetMapping(
@@ -1170,7 +1200,7 @@ class ScannerIntegration(ABC):
         """
         logger.info("Updating RegScale assets...")
         loading_assets = self._setup_progress_bar()
-        logger.info("Pre-populating cache")
+        logger.debug("Pre-populating cache")
         regscale_models.AssetMapping.populate_cache_by_plan(self.plan_id)
         regscale_models.ComponentMapping.populate_cache_by_plan(self.plan_id)
 
@@ -1200,34 +1230,126 @@ class ScannerIntegration(ABC):
 
     def _process_assets(self, assets: Iterator[IntegrationAsset], loading_assets: TaskID) -> int:
         """
-        Processes the assets using single or multi-threaded approach based on THREAD_MAX_WORKERS.
+        Process assets using single or multi-threaded approach based on THREAD_MAX_WORKERS.
 
-        :param Iterator[IntegrationAsset] assets: The assets to process
-        :param TaskID loading_assets: The task ID for the progress bar
-        :return: The number of assets processed
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param TaskID loading_assets: Task ID for the progress bar
+        :return: Number of assets processed
         :rtype: int
         """
-        assets_processed = 0
-        # prime cache
+        self._prime_asset_cache()
+        process_func = self._create_process_function(loading_assets)
+        max_workers = get_thread_workers_max()
+
+        if max_workers == 1:
+            return self._process_single_threaded(assets, process_func)
+        return self._process_multi_threaded(assets, process_func, max_workers)
+
+    def _prime_asset_cache(self) -> None:
+        """
+        Prime the asset cache by fetching assets for the given plan.
+
+        :rtype: None
+        """
         regscale_models.Asset.get_all_by_parent(
             parent_id=self.plan_id, parent_module=regscale_models.SecurityPlan.get_module_string()
         )
 
-        process_func = lambda my_asset: self._process_single_asset(my_asset, loading_assets)  # noqa: E731
+    def _create_process_function(self, loading_assets: TaskID) -> Callable[[IntegrationAsset], bool]:
+        """
+        Create a function to process a single asset.
+
+        :param TaskID loading_assets: Task ID for the progress bar
+        :return: Function that processes an asset and returns success status
+        :rtype: Callable[[IntegrationAsset], bool]
+        """
+        return lambda asset: self._process_single_asset(asset, loading_assets)
+
+    def _process_single_threaded(
+        self, assets: Iterator[IntegrationAsset], process_func: Callable[[IntegrationAsset], bool]
+    ) -> int:
+        """
+        Process assets sequentially in a single thread.
+
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :return: Number of assets processed
+        :rtype: int
+        """
+        assets_processed = 0
+        for asset in assets:
+            if process_func(asset):
+                assets_processed = self._update_processed_count(assets_processed)
+        return assets_processed
+
+    def _process_multi_threaded(
+        self, assets: Iterator[IntegrationAsset], process_func: Callable[[IntegrationAsset], bool], max_workers: int
+    ) -> int:
+        """
+        Process assets in batches using multiple threads.
+
+        :param Iterator[IntegrationAsset] assets: Assets to process
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :param int max_workers: Maximum number of worker threads
+        :return: Number of assets processed
+        :rtype: int
+        """
+        batch_size = max_workers * 2
+        assets_processed = 0
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            batch = []
+            futures = []
 
-        if get_thread_workers_max() == 1:
             for asset in assets:
-                if process_func(asset):
-                    assets_processed = self._update_processed_count(assets_processed)
-        else:
-            with concurrent.futures.ThreadPoolExecutor(max_workers=get_thread_workers_max()) as executor:
-                future_to_asset = {executor.submit(process_func, asset): asset for asset in assets}
-                for future in concurrent.futures.as_completed(future_to_asset):
-                    if future.result():
-                        assets_processed = self._update_processed_count(assets_processed)
+                batch.append(asset)
+                if len(batch) >= batch_size:
+                    assets_processed += self._submit_and_process_batch(executor, process_func, batch, futures)
+                    batch = []
+                    futures = []
+
+            if batch:  # Process any remaining items
+                assets_processed += self._submit_and_process_batch(executor, process_func, batch, futures)
 
         return assets_processed
 
+    def _submit_and_process_batch(
+        self,
+        executor: ThreadPoolExecutor,
+        process_func: Callable[[IntegrationAsset], bool],
+        batch: List[IntegrationAsset],
+        futures: List,
+    ) -> int:
+        """
+        Submit a batch of assets for processing and count successful completions.
+
+        :param ThreadPoolExecutor executor: Thread pool executor for parallel processing
+        :param Callable[[IntegrationAsset], bool] process_func: Function to process each asset
+        :param List[IntegrationAsset] batch: Batch of assets to process
+        :param List futures: List to store future objects
+        :return: Number of assets processed in this batch
+        :rtype: int
+        """
+        assets_processed = 0
+        for asset in batch:
+            futures.append(executor.submit(process_func, asset))
+
+        for future in concurrent.futures.as_completed(futures):
+            if future.result():
+                assets_processed = self._update_processed_count(assets_processed)
+
+        return assets_processed
+
+    def _update_processed_count(self, current_count: int) -> int:
+        """
+        Increment the processed count.
+
+        :param int current_count: Current number of processed items
+        :return: Updated count
+        :rtype: int
+        """
+        return current_count + 1
+
     def _process_single_asset(self, asset: IntegrationAsset, loading_assets: TaskID) -> bool:
         """
         Processes a single asset and handles any exceptions.
@@ -1258,22 +1380,52 @@ class ScannerIntegration(ABC):
             logger.info("Processed %d assets.", assets_processed)
         return assets_processed
 
+    def update_result_counts(self, key: str, results: dict[str, list]) -> None:
+        """
+        Updates the results dictionary with the given key and results.
+
+        :param str key: The key to update
+        :param dict[str, list] results: The results to update, example: ["updated": [], "created": []]
+        :rtype: None
+        """
+        if key not in self._results:
+            self._results[key] = {"created_count": 0, "updated_count": 0}
+        self._results[key]["created_count"] += len(results.get("created", []))
+        self._results[key]["updated_count"] += len(results.get("updated", []))
+
     def _perform_batch_operations(self, progress: Progress) -> None:
         """
         Performs batch operations for assets, software inventory, and data.
 
         :rtype: None
         """
-        logger.info("Bulk saving assets...")
-        regscale_models.Asset.bulk_save(progress_context=progress)
-        regscale_models.Issue.bulk_save(progress_context=progress)
-        regscale_models.Property.bulk_save(progress_context=progress)
-
+        logger.debug("Bulk saving assets...")
+        self.update_result_counts("assets", regscale_models.Asset.bulk_save(progress_context=progress))
+        logger.debug("Done bulk saving assets.")
+        logger.debug("Bulk saving issues...")
+        self.update_result_counts("issues", regscale_models.Issue.bulk_save(progress_context=progress))
+        logger.debug("Done bulk saving issues.")
+        logger.debug("Bulk saving properties...")
+        self.update_result_counts("properties", regscale_models.Property.bulk_save(progress_context=progress))
+        logger.debug("Done bulk saving properties.")
+
+        software_inventory = {}
         if self.software_to_create:
-            regscale_models.SoftwareInventory.batch_create(items=self.software_to_create, progress_context=progress)
+            logger.debug("Bulk creating software inventory...")
+            software_inventory["created_count"] = len(
+                regscale_models.SoftwareInventory.batch_create(items=self.software_to_create, progress_context=progress)
+            )
+            logger.debug("Done bulk creating software inventory.")
         if self.software_to_update:
-            regscale_models.SoftwareInventory.batch_update(items=self.software_to_update, progress_context=progress)
-        regscale_models.Data.bulk_save(progress_context=progress)
+            logger.debug("Bulk updating software inventory...")
+            software_inventory["updated_updated"] = len(
+                regscale_models.SoftwareInventory.batch_update(items=self.software_to_update, progress_context=progress)
+            )
+            logger.debug("Done bulk updating software inventory.")
+        self._results["software_inventory"] = software_inventory
+        logger.debug("Bulk saving data records...")
+        self.update_result_counts("data", regscale_models.Data.bulk_save(progress_context=progress))
+        logger.debug("Done bulk saving data records.")
 
     @staticmethod
     def get_issue_title(finding: IntegrationFinding) -> str:
@@ -1389,7 +1541,7 @@ class ScannerIntegration(ABC):
         issue.issueOwnerId = self.assessor_id
         issue.securityPlanId = self.plan_id
         issue.identification = "Vulnerability Assessment"
-        issue.dateFirstDetected = finding.date_created
+        issue.dateFirstDetected = finding.first_seen
         issue.dueDate = finding.due_date
         issue.description = description
         issue.sourceReport = finding.source_report or self.title
@@ -1803,9 +1955,8 @@ class ScannerIntegration(ABC):
         scan_history = self.create_scan_history()
         current_vulnerabilities: Dict[int, Set[int]] = defaultdict(set)
         processed_findings_count = 0
-        findings_to_process = self.num_findings_to_process
         loading_findings = self.finding_progress.add_task(
-            f"[#f8b737]Processing {f'{findings_to_process} ' if findings_to_process else ''}finding(s) from {self.title}",
+            f"[#f8b737]Processing {f'{self.num_findings_to_process} ' if self.num_findings_to_process else ''}finding(s) from {self.title}",
             total=self.num_findings_to_process if self.num_findings_to_process else None,
         )
 
@@ -1824,13 +1975,11 @@ class ScannerIntegration(ABC):
                 self.process_finding(finding_to_process, scan_history, current_vulnerabilities)
                 with count_lock:
                     processed_findings_count += 1
-                    if findings_to_process and self.finding_progress.tasks[loading_findings].total != float(
-                        findings_to_process
-                    ):
+                    if self.num_findings_to_process:
                         self.finding_progress.update(
                             loading_findings,
-                            total=findings_to_process,
-                            description=f"[#f8b737]Processing {findings_to_process} findings from {self.title}.",
+                            total=self.num_findings_to_process,
+                            description=f"[#f8b737]Processing {self.num_findings_to_process} findings from {self.title}.",
                         )
                     self.finding_progress.advance(loading_findings, 1)
             except Exception as exc:
@@ -1844,12 +1993,25 @@ class ScannerIntegration(ABC):
             for finding in findings:
                 process_finding_with_progress(finding)
         else:
+            # Process findings in batches to control memory usage
+            batch_size = get_thread_workers_max() * 2  # Set batch size based on thread count
             with concurrent.futures.ThreadPoolExecutor(max_workers=get_thread_workers_max()) as executor:
-                list(executor.map(process_finding_with_progress, findings))
+                batch = []
+                for finding in findings:
+                    batch.append(finding)
+                    if len(batch) >= batch_size:
+                        # Process this batch
+                        list(executor.map(process_finding_with_progress, batch))
+                        # Clear the batch
+                        batch = []
+
+                # Process any remaining items
+                if batch:
+                    list(executor.map(process_finding_with_progress, batch))
 
         # Close outdated issues
-        scan_history.save()
-        regscale_models.Issue.bulk_save(progress_context=self.finding_progress)
+        self._results["scan_history"] = scan_history.save()
+        self.update_result_counts("issues", regscale_models.Issue.bulk_save(progress_context=self.finding_progress))
         self.close_outdated_issues(current_vulnerabilities)
 
         return processed_findings_count
@@ -2027,6 +2189,9 @@ class ScannerIntegration(ABC):
                 finding.vpr_score if hasattr(finding, "vprScore") else None
             ),  # If this is the VPR score, otherwise use a different field
             cvsSv3BaseScore=finding.cvss_v3_base_score or finding.cvss_v3_score or finding.cvss_score,
+            cvsSv2BaseScore=finding.cvss_v2_score,
+            cvsSv3BaseVector=finding.cvss_v3_vector,
+            cvsSv2BaseVector=finding.cvss_v2_vector,
             scanId=scan_history.id,
             severity=self.issue_to_vulnerability_map.get(finding.severity, regscale_models.VulnerabilitySeverity.Low),
             description=finding.description,
@@ -2039,29 +2204,36 @@ class ScannerIntegration(ABC):
             firstSeen=finding.first_seen,
             lastSeen=finding.last_seen,
             plugInName=finding.cve or finding.plugin_name,  # Use CVE if available, otherwise use plugin name
-            # plugInId=finding.plugin_id,  # Vulnerability.pluginId is an int, but it is a string on Issue
+            plugInId=finding.plugin_id,
             exploitAvailable=None,  # Set this if you have information about exploit availability
             plugInText=finding.observations,  # or finding.evidence, whichever is more appropriate
             port=finding.port if hasattr(finding, "port") else None,
             protocol=finding.protocol if hasattr(finding, "protocol") else None,
             operatingSystem=asset.operating_system if hasattr(asset, "operating_system") else None,
+            fixedVersions=finding.fixed_versions,
+            buildVersion=finding.build_version,
+            fixStatus=finding.fix_status,
+            installedVersions=finding.installed_versions,
+            affectedOS=finding.affected_os,
+            packagePath=finding.package_path,
+            imageDigest=finding.image_digest,
+            affectedPackages=finding.affected_packages,
         )
 
         vulnerability = vulnerability.create_or_update()
-        if re.match(r"^\d+\.\d+(\.\d+){0,2}$", self.regscale_version) or self.regscale_version >= "5.64.0":
-            regscale_models.VulnerabilityMapping(
-                vulnerabilityId=vulnerability.id,
-                assetId=asset.id,
-                scanId=scan_history.id,
-                securityPlansId=self.plan_id,
-                createdById=self.assessor_id,
-                tenantsId=self.tenant_id,
-                isPublic=True,
-                dateCreated=get_current_datetime(),
-                firstSeen=finding.first_seen,
-                lastSeen=finding.last_seen,
-                status=finding.status,
-            ).create_unique()
+        regscale_models.VulnerabilityMapping(
+            vulnerabilityId=vulnerability.id,
+            assetId=asset.id,
+            scanId=scan_history.id,
+            securityPlansId=self.plan_id,
+            createdById=self.assessor_id,
+            tenantsId=self.tenant_id,
+            isPublic=True,
+            dateCreated=get_current_datetime(),
+            firstSeen=finding.first_seen,
+            lastSeen=finding.last_seen,
+            status=finding.status,
+        ).create_unique()
         return vulnerability
 
     def handle_vulnerability(
@@ -2205,7 +2377,9 @@ class ScannerIntegration(ABC):
         )
 
         # Create a progress bar
-        task_id = self.finding_progress.add_task("[cyan]Closing outdated issues...", total=len(open_issues))
+        task_id = self.finding_progress.add_task(
+            f"[cyan]Analyzing {len(open_issues)} issue(s) and closing any outdated issue(s)...", total=len(open_issues)
+        )
 
         for issue in open_issues:
             if self.should_close_issue(issue, current_vulnerabilities):
@@ -2230,7 +2404,10 @@ class ScannerIntegration(ABC):
         for control_id in affected_control_ids:
             self.update_control_implementation_status_after_close(control_id)
 
-        logger.info("Closed %d outdated issues.", closed_count)
+        if closed_count > 0:
+            logger.info("Closed %d outdated issues.", closed_count)
+        else:
+            logger.info("No outdated issues to close.")
         return closed_count
 
     def update_control_implementation_status_after_close(self, control_id: int) -> None:
@@ -2370,7 +2547,7 @@ class ScannerIntegration(ABC):
         :return: The number of findings processed
         :rtype: int
         """
-        logger.info("Syncing %s findings...", cls.title)
+        logger.info("Syncing %s findings...", kwargs.get("title", cls.title))
         instance = cls(plan_id=plan_id)
         instance.set_keys(**kwargs)
         # If a progress object was passed, use it instead of creating a new one
@@ -2394,7 +2571,25 @@ class ScannerIntegration(ABC):
         else:
             logger.info("All findings have been processed successfully.")
 
-        logger.info("Processed %d findings.", findings_processed)
+        if scan_history := instance._results.get("scan_history"):
+            logger.info(
+                "Processed %d findings: %d Critical(s), %d High(s), %d Moderate(s), %d Low(s).",
+                findings_processed,
+                scan_history.vCritical,
+                scan_history.vHigh,
+                scan_history.vMedium,
+                scan_history.vLow,
+            )
+        else:
+            logger.info("Processed %d findings.", findings_processed)
+        issue_created_count = instance._results.get("issues", {}).get("created_count", 0)
+        issue_updated_count = instance._results.get("issues", {}).get("updated_count", 0)
+        if issue_created_count or issue_updated_count:
+            logger.info(
+                "Created %d issue(s) and updated %d issue(s) in RegScale.",
+                issue_created_count,
+                issue_updated_count,
+            )
         return findings_processed
 
     @classmethod
@@ -2406,7 +2601,7 @@ class ScannerIntegration(ABC):
         :return: The number of assets processed
         :rtype: int
         """
-        logger.info("Syncing %s assets...", cls.title)
+        logger.info("Syncing %s assets...", kwargs.get("title", cls.title))
         instance = cls(plan_id=plan_id, **kwargs)
         instance.set_keys(**kwargs)
         instance.asset_progress = kwargs.pop("progress") if "progress" in kwargs else create_progress_object()
@@ -2425,7 +2620,16 @@ class ScannerIntegration(ABC):
             logger.info("All assets have been processed successfully.")
 
         APIHandler().log_api_summary()
-        logger.info("%d assets processed.", assets_processed)
+        created_count = instance._results.get("assets", {}).get("created_count", 0)
+        updated_count = instance._results.get("assets", {}).get("updated_count", 0)
+        dedupe_count = assets_processed - (created_count + updated_count)
+        logger.info(
+            "%d assets processed and %d asset(s) deduped. %d asset(s) created & %d asset(s) updated in RegScale.",
+            assets_processed,
+            dedupe_count,
+            created_count,
+            updated_count,
+        )
         return assets_processed
 
     @classmethod

regscale/models/app_models/mapping.py

@@ -226,7 +226,7 @@ class Mapping(BaseModel):
         """
         return self.mapping.get(key)
 
-    def get_value(self, dat: Optional[dict], key: str, default_val: Optional[Any] = "", warnings: bool = True) -> Any:
+    def get_value(self, dat: Optional[dict], key: str, default_val: Optional[Any] = "") -> Any:
         """
         Get the value from a dictionary by mapped key
 
@@ -241,8 +241,8 @@ class Mapping(BaseModel):
         if key == "None" or key is None:
             return default_val
         mapped_key = self.mapping.get(key)
-        if not mapped_key and warnings:
-            self._logger.warning(f"Value for key '{key}' not found in mapping.")
+        if not mapped_key:
+            self._logger.debug(f"Value for key '{key}' not found in mapping.")
         if dat and mapped_key:
             val = dat.get(mapped_key)
             if isinstance(val, str):

regscale/models/integration_models/amazon_models/inspector.py

@@ -129,21 +129,21 @@ class InspectorRecord(BaseModel):
         platform_key = list(details.keys())[0] if details.keys() else None
 
         return InspectorRecord(
-            aws_account_id=mapping.get_value(finding, "awsAccountId", "", warnings=False),
-            description=mapping.get_value(finding, "description", warnings=False),
-            exploit_available=mapping.get_value(finding, "exploitAvailable", warnings=False),
-            finding_arn=mapping.get_value(finding, "findingArn", warnings=False),
-            first_seen=mapping.get_value(finding, "firstObservedAt", warnings=False),
-            fix_available=mapping.get_value(finding, "fixAvailable", warnings=False),
-            last_seen=mapping.get_value(finding, "lastObservedAt", warnings=False),
+            aws_account_id=mapping.get_value(finding, "awsAccountId", ""),
+            description=mapping.get_value(finding, "description"),
+            exploit_available=mapping.get_value(finding, "exploitAvailable"),
+            finding_arn=mapping.get_value(finding, "findingArn"),
+            first_seen=mapping.get_value(finding, "firstObservedAt"),
+            fix_available=mapping.get_value(finding, "fixAvailable"),
+            last_seen=mapping.get_value(finding, "lastObservedAt"),
             remediation=mapping.get_value(finding, "remediation", {}).get("recommendation", {}).get("text", ""),
-            severity=mapping.get_value(finding, "Severity", warnings=False),
-            status=mapping.get_value(finding, "status", warnings=False),
-            title=mapping.get_value(finding, "title", warnings=False),
+            severity=mapping.get_value(finding, "Severity"),
+            status=mapping.get_value(finding, "status"),
+            title=mapping.get_value(finding, "title"),
             resource_type=resource.get("type"),
             resource_id=resource.get("id"),
             region=resource.get("region"),
-            last_updated=mapping.get_value(finding, "updatedAt", warnings=False),
+            last_updated=mapping.get_value(finding, "updatedAt"),
             platform=resource.get("details", {}).get(platform_key, {}).get("platform", ""),
             resource_tags=" ,".join(resource.get("details", {}).get(platform_key, {}).get("imageTags", "")),
             affected_packages=cls.get_vulnerable_package_info(vulnerabilities, "name"),
@@ -152,18 +152,16 @@ class InspectorRecord(BaseModel):
             package_remediation=cls.get_vulnerable_package_info(vulnerabilities, "remediation"),
             vulnerability_id=vulnerabilities.get("vulnerabilityId") if vulnerabilities else None,
             vendor=vulnerabilities.get("source") if vulnerabilities else None,
-            vendor_severity=mapping.get_value(finding, "severity", warnings=False),
+            vendor_severity=mapping.get_value(finding, "severity"),
             vendor_advisory=vulnerabilities.get("sourceUrl") if vulnerabilities else None,
             vendor_advisory_published=vulnerabilities.get("vendorCreatedAt") if vulnerabilities else None,
             package_manager=cls.get_vulnerable_package_info(
-                mapping.get_value(finding, "packageVulnerabilityDetails", {}, warnings=False), "packageManager"
+                mapping.get_value(finding, "packageVulnerabilityDetails", {}), "packageManager"
             ),
             file_path=cls.get_vulnerable_package_info(
-                mapping.get_value(finding, "packageVulnerabilityDetails", {}, warnings=False), "filePath"
-            ),
-            reference_urls=mapping.get_value(finding, "packageVulnerabilityDetails", {}, warnings=False).get(
-                "sourceUrl"
+                mapping.get_value(finding, "packageVulnerabilityDetails", {}), "filePath"
             ),
+            reference_urls=mapping.get_value(finding, "packageVulnerabilityDetails", {}).get("sourceUrl"),
         )
 
     @classmethod

regscale/models/integration_models/aqua.py

@@ -209,11 +209,7 @@ class Aqua(FlatFileImporter):
             self.nvd_cvss_v2_severity,
             self.vendor_cvss_v3_severity,
             # This field may or may not be available in the file (Coalfire has it, BMC does not.)
-            (
-                self.vendor_cvss_v2_severity
-                if self.mapping.get_value(dat, self.vendor_cvss_v2_severity, warnings=False)
-                else None
-            ),
+            (self.vendor_cvss_v2_severity if self.mapping.get_value(dat, self.vendor_cvss_v2_severity) else None),
         ]
         severity = "info"
         for key in precedence_order: