regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/integrations/commercial/qualys/scanner.py

@@ -4,6 +4,7 @@ Qualys Total Cloud scanner integration class using JSONLScannerIntegration.
 
 import logging
 import os
+import threading
 import time
 import traceback
 import xml.etree.ElementTree as ET
@@ -27,18 +28,20 @@ from regscale.integrations.scanner_integration import (
 )
 from regscale.integrations.variables import ScannerVariables
 from regscale.models import AssetStatus, IssueSeverity, IssueStatus
+from regscale import models as regscale_models
 
 logger = logging.getLogger("regscale")
 
 NO_RESULTS = "No results available"
 NO_DESCRIPTION = "No description available"
 NO_REMEDIATION = "No remediation information available"
+SCANNING_TOOL_NAME = "Qualys Total Cloud"
 
 
 class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
     """Class for handling Qualys Total Cloud scanner integration using JSONL."""
 
-    title: str = "Qualys Total Cloud"
+    title: str = SCANNING_TOOL_NAME
     asset_identifier_field: str = "qualysId"
     finding_severity_map: Dict[str, Any] = {
         "0": IssueSeverity.NotAssigned.value,
@@ -73,34 +76,52 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
         self.xml_data = kwargs.pop("xml_data", None)
         self.containers = kwargs.pop("containers", None)
         self.is_component = kwargs.get("is_component", False)
-        # Setting a dummy file path to avoid validation errors
+
+        self._setup_file_path(kwargs)
+        self._apply_vulnerability_creation_setting(kwargs)
+        self._apply_ssl_verification_setting(kwargs)
+        self._apply_thread_workers_setting(kwargs)
+
+        super().__init__(*args, **kwargs)
+        # No need to initialize clients, they are inherited from the parent class
+
+    def _setup_file_path(self, kwargs: Dict[str, Any]) -> None:
+        """Setup file path for XML data processing."""
         if self.xml_data and "file_path" not in kwargs:
             kwargs["file_path"] = None
 
-        # Apply ScannerVariables settings
-        if not kwargs.get("vulnerability_creation"):
-            # Check QualysVariables-specific override first
-            if hasattr(QualysVariables, "vulnerabilityCreation") and QualysVariables.vulnerabilityCreation:
-                kwargs["vulnerability_creation"] = QualysVariables.vulnerabilityCreation
-                logger.info(f"Using Qualys-specific vulnerability creation mode: {kwargs['vulnerability_creation']}")
-            # Use global ScannerVariables if no Qualys-specific setting
-            elif hasattr(ScannerVariables, "vulnerabilityCreation"):
-                kwargs["vulnerability_creation"] = ScannerVariables.vulnerabilityCreation
-                logger.info(f"Using global vulnerability creation mode: {kwargs['vulnerability_creation']}")
-
-        # Apply SSL verification setting from ScannerVariables
+    def _apply_vulnerability_creation_setting(self, kwargs: Dict[str, Any]) -> None:
+        """Apply vulnerability creation setting from variables."""
+        if kwargs.get("vulnerability_creation"):
+            return
+
+        if self._has_qualys_vulnerability_creation():
+            kwargs["vulnerability_creation"] = QualysVariables.vulnerabilityCreation
+            logger.info(f"Using Qualys-specific vulnerability creation mode: {kwargs['vulnerability_creation']}")
+        elif self._has_scanner_vulnerability_creation():
+            kwargs["vulnerability_creation"] = ScannerVariables.vulnerabilityCreation
+            logger.info(f"Using global vulnerability creation mode: {kwargs['vulnerability_creation']}")
+
+    def _has_qualys_vulnerability_creation(self) -> bool:
+        """Check if QualysVariables has vulnerability creation setting."""
+        return hasattr(QualysVariables, "vulnerabilityCreation") and QualysVariables.vulnerabilityCreation
+
+    def _has_scanner_vulnerability_creation(self) -> bool:
+        """Check if ScannerVariables has vulnerability creation setting."""
+        return hasattr(ScannerVariables, "vulnerabilityCreation")
+
+    def _apply_ssl_verification_setting(self, kwargs: Dict[str, Any]) -> None:
+        """Apply SSL verification setting from ScannerVariables."""
         if not kwargs.get("ssl_verify") and hasattr(ScannerVariables, "sslVerify"):
             kwargs["ssl_verify"] = ScannerVariables.sslVerify
             logger.debug(f"Using SSL verification setting: {kwargs['ssl_verify']}")
 
-        # Apply ScannerVariables.threadMaxWorkers if available
+    def _apply_thread_workers_setting(self, kwargs: Dict[str, Any]) -> None:
+        """Apply thread max workers setting from ScannerVariables."""
         if not kwargs.get("max_workers") and hasattr(ScannerVariables, "threadMaxWorkers"):
             kwargs["max_workers"] = ScannerVariables.threadMaxWorkers
             logger.debug(f"Using thread max workers: {kwargs['max_workers']}")
 
-        super().__init__(*args, **kwargs)
-        # No need to initialize clients, they are inherited from the parent class
-
     def is_valid_file(self, data: Any, file_path: Union[Path, str]) -> Tuple[bool, Optional[Dict[str, Any]]]:
         """
         Check if the XML data is valid for Qualys Total Cloud.
@@ -230,6 +251,12 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
         # Extract host information
         host_info = self._extract_host_information(processed_host)
 
+        # Log asset creation for debugging
+        logger.debug(f"Creating asset for host ID: {host_info['host_id']}")
+        logger.debug(f"Asset name: {host_info['name']}")
+        logger.debug(f"Plan ID: {self.plan_id}, Parent Module: {self.parent_module}")
+        logger.debug(f"Is Component: {self.is_component}")
+
         # Create and return the asset
         return IntegrationAsset(
             name=host_info["name"],
@@ -448,6 +475,23 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
 
         current_time = self.scan_date or get_current_datetime()
 
+        # Extract CVSS scores and convert to float if possible
+        cvss_v3_score = detection.get("CVSS3_BASE")
+        cvss_v2_score = detection.get("CVSS_BASE")
+
+        # Convert CVSS scores to float if they're strings
+        if cvss_v3_score and isinstance(cvss_v3_score, str):
+            try:
+                cvss_v3_score = float(cvss_v3_score)
+            except (ValueError, TypeError):
+                cvss_v3_score = None
+
+        if cvss_v2_score and isinstance(cvss_v2_score, str):
+            try:
+                cvss_v2_score = float(cvss_v2_score)
+            except (ValueError, TypeError):
+                cvss_v2_score = None
+
         return {
             "qid": detection.get("QID", "Unknown"),
             "severity": detection.get("SEVERITY", "0"),
@@ -456,9 +500,9 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
             "last_found": detection.get("LAST_FOUND_DATETIME", current_time),
             "unique_id": detection.get("UNIQUE_VULN_ID", f"QID-{detection.get('QID', 'Unknown')}"),
             "results": detection.get("RESULTS", NO_RESULTS),
-            "cvss_v3_score": detection.get("CVSS3_BASE"),
+            "cvss_v3_score": cvss_v3_score,
             "cvss_v3_vector": detection.get("CVSS3_VECTOR", ""),
-            "cvss_v2_score": detection.get("CVSS_BASE"),
+            "cvss_v2_score": cvss_v2_score,
             "cvss_v2_vector": detection.get("CVSS_VECTOR", ""),
         }
 
@@ -504,29 +548,79 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
         if not detection:
             return ""
 
-        cve_id = ""
         try:
-            cve_list = detection.get("CVE_ID_LIST", {})
-            if not cve_list:
-                return ""
-
-            if not isinstance(cve_list, dict):
-                logger.warning(f"Expected dictionary for CVE_ID_LIST, got {type(cve_list)}")
-                return ""
-
-            if "CVE_ID" in cve_list:
-                cve_data = cve_list.get("CVE_ID", [])
-                if isinstance(cve_data, list) and cve_data:
-                    cve_id = str(cve_data[0]) if cve_data[0] else ""
-                elif isinstance(cve_data, str):
-                    cve_id = cve_data
-                elif cve_data:
-                    # Try to convert to string if it's something else
-                    cve_id = str(cve_data)
+            # Try to extract CVE from CVE_ID_LIST first
+            cve_id = self._extract_cve_from_cve_list(detection)
+            if cve_id:
+                return cve_id
+
+            # Try direct CVE fields if CVE_ID_LIST didn't work
+            cve_id = self._extract_cve_from_direct_fields(detection)
+            if cve_id:
+                return cve_id
+
         except Exception as e:
             logger.warning(f"Error extracting CVE_ID: {str(e)}")
 
-        return cve_id
+        return ""
+
+    def _extract_cve_from_cve_list(self, detection: Dict[str, Any]) -> str:
+        """
+        Extract CVE ID from CVE_ID_LIST field.
+
+        :param Dict[str, Any] detection: Detection data
+        :return: CVE ID string
+        :rtype: str
+        """
+        cve_list = detection.get("CVE_ID_LIST", {})
+        if not cve_list:
+            return ""
+
+        if not isinstance(cve_list, dict):
+            logger.warning(f"Expected dictionary for CVE_ID_LIST, got {type(cve_list)}")
+            return ""
+
+        if "CVE_ID" not in cve_list:
+            return ""
+
+        cve_data = cve_list.get("CVE_ID", [])
+        return self._convert_cve_data_to_string(cve_data)
+
+    def _extract_cve_from_direct_fields(self, detection: Dict[str, Any]) -> str:
+        """
+        Extract CVE ID from direct CVE fields.
+
+        :param Dict[str, Any] detection: Detection data
+        :return: CVE ID string
+        :rtype: str
+        """
+        # Try CVE field directly
+        cve_id = detection.get("CVE", "")
+        if cve_id:
+            return str(cve_id)
+
+        # Try CVE_ID field directly
+        cve_id = detection.get("CVE_ID", "")
+        if cve_id:
+            return str(cve_id)
+
+        return ""
+
+    def _convert_cve_data_to_string(self, cve_data: Any) -> str:
+        """
+        Convert CVE data to string format.
+
+        :param Any cve_data: CVE data to convert
+        :return: CVE ID string
+        :rtype: str
+        """
+        if isinstance(cve_data, list) and cve_data:
+            return str(cve_data[0]) if cve_data[0] else ""
+        elif isinstance(cve_data, str):
+            return cve_data
+        elif cve_data:
+            return str(cve_data)
+        return ""
 
     def _extract_cve_id_from_xml(self, detection: Optional[Union[Dict[str, Any], ET.Element]]) -> str:
         """
@@ -648,6 +742,12 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
         current_time = self.scan_date or get_current_datetime()
         qid = finding_data.get("qid", "Unknown")
 
+        # Log the finding data for debugging
+        logger.debug(f"Creating finding for QID {qid}, host {host_id}")
+        logger.debug(f"CVE: {finding_data.get('cve_id', 'None')}")
+        logger.debug(f"CVSS V3 Score: {finding_data.get('cvss_v3_score', 'None')}")
+        logger.debug(f"CVSS V2 Score: {finding_data.get('cvss_v2_score', 'None')}")
+
         return IntegrationFinding(
             title=finding_data.get("title", f"Qualys Vulnerability QID-{qid}"),
             description=finding_data.get("diagnosis", NO_DESCRIPTION),
@@ -1376,3 +1476,169 @@ class QualysTotalCloudJSONLIntegration(JSONLScannerIntegration):
 
         # Default to Open for any unknown status
         return IssueStatus.Open
+
+    def create_vulnerability_from_finding(
+        self, finding: IntegrationFinding, asset: regscale_models.Asset, scan_history: regscale_models.ScanHistory
+    ) -> regscale_models.Vulnerability:
+        """
+        Override the parent method to add better debugging and ensure proper vulnerability mapping creation.
+
+        :param IntegrationFinding finding: The integration finding
+        :param regscale_models.Asset asset: The associated asset
+        :param regscale_models.ScanHistory scan_history: The scan history
+        :return: The created vulnerability
+        :rtype: regscale_models.Vulnerability
+        """
+        logger.debug(f"Creating vulnerability from finding: {finding.title}")
+        logger.debug(f"Asset ID: {asset.id}, Asset Name: {asset.name}")
+        logger.debug(f"Scan History ID: {scan_history.id}")
+        logger.debug(f"Plan ID: {self.plan_id}, Parent Module: {self.parent_module}")
+        logger.debug(f"Is Component: {self.is_component}")
+
+        # Call the parent method
+        vulnerability = super().create_vulnerability_from_finding(finding, asset, scan_history)
+
+        logger.debug(f"Created vulnerability with ID: {vulnerability.id}")
+        logger.debug(f"Vulnerability parentId: {vulnerability.parentId}")
+        logger.debug(f"Vulnerability parentModule: {vulnerability.parentModule}")
+
+        # Verify the vulnerability mapping was created
+        try:
+            mappings = regscale_models.VulnerabilityMapping.find_by_vulnerability(vulnerability.id)
+            logger.debug(f"Found {len(mappings)} vulnerability mappings for vulnerability {vulnerability.id}")
+            for mapping in mappings:
+                logger.debug(
+                    f"Mapping - Asset ID: {mapping.assetId}, Scan ID: {mapping.scanId}, Security Plan ID: {mapping.securityPlanId}"
+                )
+        except Exception as e:
+            logger.warning(f"Error checking vulnerability mappings: {e}")
+
+        return vulnerability
+
+    def handle_vulnerability(
+        self,
+        finding: IntegrationFinding,
+        asset: Optional[regscale_models.Asset],
+        scan_history: regscale_models.ScanHistory,
+    ) -> Optional[int]:
+        """
+        Override parent method to ensure Qualys findings always create vulnerabilities.
+        This ensures that Qualys vulnerabilities are properly populated in RegScale.
+
+        :param IntegrationFinding finding: The integration finding
+        :param Optional[regscale_models.Asset] asset: The associated asset
+        :param regscale_models.ScanHistory scan_history: The scan history
+        :rtype: Optional[int]
+        :return: The vulnerability ID
+        """
+        # Check for required fields - either plugin_name or cve must be present
+        if not (finding.plugin_name or finding.cve):
+            logger.warning(
+                f"Qualys: Skipping vulnerability creation - missing plugin_name and cve for finding {finding.external_id}"
+            )
+            return None
+
+        # Ensure vulnerability creation is enabled for Qualys
+        logger.debug(f"Qualys: Vulnerability creation setting: {self.vulnerability_creation}")
+        if self.vulnerability_creation == "NoIssue":
+            logger.debug(f"Qualys: Vulnerability creation disabled, skipping finding {finding.external_id}")
+            return None
+
+        # Create vulnerability using parent method
+        logger.debug(f"Qualys: Calling parent handle_vulnerability for finding {finding.external_id}")
+        vulnerability_id = super().handle_vulnerability(finding, asset, scan_history)
+
+        if vulnerability_id:
+            logger.debug(f"Qualys: Created vulnerability {vulnerability_id} for finding {finding.external_id}")
+        else:
+            logger.warning(f"Qualys: Failed to create vulnerability for finding {finding.external_id}")
+
+        return vulnerability_id
+
+    def set_severity_count_for_scan(
+        self, severity: str, scan_history: regscale_models.ScanHistory, lock: Optional[threading.RLock] = None
+    ) -> None:
+        """
+        Override parent method to ensure Qualys scan history severity counts are properly updated.
+        This ensures that the vulnerability counts are accurately reflected in the scan history.
+
+        :param str severity: Severity of the vulnerability
+        :param regscale_models.ScanHistory scan_history: Scan history object
+        :param Optional[threading.RLock] lock: Thread lock for synchronization
+        :rtype: None
+        """
+        # Use parent method to update severity counts with thread-safe locking
+        # Pass lock if provided, otherwise use our instance lock
+        super().set_severity_count_for_scan(severity, scan_history, lock or self.scan_history_lock)
+
+    def create_scan_history(self) -> regscale_models.ScanHistory:
+        """
+        Override parent method to ensure Qualys scan history is properly created.
+        This ensures that the scanning tool name is correctly set for Qualys scans.
+        Also reuses existing scan history records for the same day and tool to avoid duplicates.
+
+        :return: A newly created or reused ScanHistory object
+        :rtype: regscale_models.ScanHistory
+        """
+        logger.debug(f"Creating scan history for plan {self.plan_id}, module {self.parent_module}")
+
+        try:
+            # Load existing scans for the plan/module
+            existing_scans = regscale_models.ScanHistory.get_all_by_parent(
+                parent_id=self.plan_id, parent_module=self.parent_module
+            )
+
+            # Normalize target date to date component only
+            target_dt = self.scan_date if self.scan_date else get_current_datetime()
+            target_date_only = target_dt.split("T")[0] if isinstance(target_dt, str) else str(target_dt)[:10]
+
+            # Find an existing scan for today and this tool
+            for scan in existing_scans:
+                try:
+                    if getattr(scan, "scanningTool", None) == SCANNING_TOOL_NAME and getattr(scan, "scanDate", None):
+                        scan_date = str(scan.scanDate)
+                        scan_date_only = scan_date.split("T")[0]
+                        if scan_date_only == target_date_only:
+                            # Reuse this scan history; refresh last updated
+                            logger.debug(f"Reusing existing scan history {scan.id} for {target_date_only}")
+                            scan.dateLastUpdated = get_current_datetime()
+                            scan.lastUpdatedById = self.assessor_id
+                            scan.save()
+                            return scan
+                except Exception:
+                    # Skip any malformed scan records
+                    continue
+
+            # No existing same-day scan found, create new
+            logger.debug("No existing scan history found for today, creating new one")
+            scan_history = regscale_models.ScanHistory(
+                parentId=self.plan_id,
+                parentModule=self.parent_module,
+                scanningTool=SCANNING_TOOL_NAME,  # Ensure proper scanning tool name
+                scanDate=self.scan_date if self.scan_date else get_current_datetime(),
+                createdById=self.assessor_id,
+                lastUpdatedById=self.assessor_id,
+                tenantsId=self.tenant_id,
+                vLow=0,
+                vMedium=0,
+                vHigh=0,
+                vCritical=0,
+            ).create()
+
+            logger.debug(f"Created new scan history with ID: {scan_history.id}")
+
+            # Ensure the scan history is properly created and cached
+            count = 0
+            regscale_models.ScanHistory.delete_object_cache(scan_history)
+            while not regscale_models.ScanHistory.get_object(object_id=scan_history.id) or count > 10:
+                logger.info("Waiting for ScanHistory to be created...")
+                time.sleep(1)
+                count += 1
+                regscale_models.ScanHistory.delete_object_cache(scan_history)
+
+            return scan_history
+
+        except Exception as e:
+            logger.error(f"Error in create_scan_history: {e}")
+            # Fallback: create new scan history using parent method
+            return super().create_scan_history()