regscale-cli 6.20.9.1__py3-none-any.whl → 6.21.0.0__py3-none-any.whl

regscale/integrations/scanner_integration.py

@@ -28,7 +28,7 @@ from regscale.integrations.commercial.durosuite.variables import DuroSuiteVariab
 from regscale.integrations.commercial.stig_mapper_integration.mapping_engine import StigMappingEngine as STIGMapper
 from regscale.integrations.public.cisa import pull_cisa_kev
 from regscale.integrations.variables import ScannerVariables
-from regscale.models import DateTimeEncoder, OpenIssueDict, regscale_models
+from regscale.models import DateTimeEncoder, OpenIssueDict, Property, regscale_models
 from regscale.utils.threading import ThreadSafeDict, ThreadSafeList
 
 logger = logging.getLogger(__name__)
@@ -407,7 +407,7 @@ class IntegrationFinding:
     issue_type: str = "Risk"
     date_created: str = dataclasses.field(default_factory=get_current_datetime)
     date_last_updated: str = dataclasses.field(default_factory=get_current_datetime)
-    due_date: str = dataclasses.field(default_factory=lambda: date_str(days_from_today(60)))
+    due_date: str = ""  # dataclasses.field(default_factory=lambda: date_str(days_from_today(60)))
     external_id: str = ""
     gaps: str = ""
     observations: str = ""
@@ -463,6 +463,9 @@ class IntegrationFinding:
     # Additional fields from Wiz integration
     vpr_score: Optional[float] = None
 
+    # Extra data field for miscellaneous data
+    extra_data: Dict[str, Any] = dataclasses.field(default_factory=dict)
+
     def __post_init__(self):
         """Validate and adjust types after initialization."""
         # Set default date values if empty
@@ -1646,25 +1649,34 @@ class ScannerIntegration(ABC):
         issue.dateLastUpdated = get_current_datetime()
 
         if finding.cve:
-            issue = self.lookup_kev_and_upate_issue(cve=finding.cve, issue=issue, cisa_kevs=self._kev_data)
+            issue = self.lookup_kev_and_update_issue(cve=finding.cve, issue=issue, cisa_kevs=self._kev_data)
 
         if existing_issue:
-            logger.debug("Saving existing issue %s with assetIdentifier: %s", issue.id, issue.assetIdentifier)
+            logger.debug("Saving Old Issue: %s  with assetIdentifier: %s", issue.id, issue.assetIdentifier)
             issue.save(bulk=True)
+            logger.debug("Saved existing issue %s with assetIdentifier: %s", issue.id, issue.assetIdentifier)
+
         else:
             issue = issue.create_or_update(
                 bulk_update=True, defaults={"otherIdentifier": self._get_other_identifier(finding, is_poam)}
             )
+            self.extra_data_to_properties(finding, issue.id)
 
-        self._handle_property_creation_for_issue(issue, finding)
+        self._handle_property_and_milestone_creation(issue, finding, existing_issue)
         return issue
 
-    def _handle_property_creation_for_issue(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
+    def _handle_property_and_milestone_creation(
+        self,
+        issue: regscale_models.Issue,
+        finding: IntegrationFinding,
+        existing_issue: Optional[regscale_models.Issue] = None,
+    ) -> None:
         """
         Handles property creation for an issue based on the finding data
 
         :param regscale_models.Issue issue: The issue to handle properties for
         :param IntegrationFinding finding: The finding data
+        :param bool new_issue: Whether this is a new issue
         :rtype: None
         """
         if poc := finding.point_of_contact:
@@ -1685,6 +1697,74 @@ class ScannerIntegration(ABC):
             ).create_or_update()
             logger.debug("Added CWE property %s to issue %s", finding.plugin_id, issue.id)
 
+        if ScannerVariables.useMilestones:
+            if (
+                existing_issue
+                and existing_issue.status == regscale_models.IssueStatus.Closed
+                and issue.status == regscale_models.IssueStatus.Open
+            ):
+                regscale_models.Milestone(
+                    title=f"Issue reopened from {self.title} scan",
+                    milestoneDate=get_current_datetime(),
+                    responsiblePersonId=self.assessor_id,
+                    parentID=issue.id,
+                    parentModule="issues",
+                ).create_or_update()
+                logger.debug("Added milestone for issue %s from finding %s", issue.id, finding.external_id)
+            elif (
+                existing_issue
+                and existing_issue.status == regscale_models.IssueStatus.Open
+                and issue.status == regscale_models.IssueStatus.Closed
+            ):
+                regscale_models.Milestone(
+                    title=f"Issue closed from {self.title} scan",
+                    milestoneDate=issue.dateCompleted,
+                    responsiblePersonId=self.assessor_id,
+                    parentID=issue.id,
+                    parentModule="issues",
+                ).create_or_update()
+                logger.debug("Added milestone for issue %s from finding %s", issue.id, finding.external_id)
+            elif not existing_issue:
+                regscale_models.Milestone(
+                    title=f"Issue created from {self.title} scan",
+                    milestoneDate=self.scan_date,
+                    responsiblePersonId=self.assessor_id,
+                    parentID=issue.id,
+                    parentModule="issues",
+                ).create_or_update()
+                logger.debug("Created milestone for issue %s from finding %s", issue.id, finding.external_id)
+            else:
+                logger.debug("No milestone created for issue %s from finding %s", issue.id, finding.external_id)
+
+    @staticmethod
+    def extra_data_to_properties(finding: IntegrationFinding, issue_id: int) -> None:
+        """
+        Adds extra data to properties for an issue in a separate thread
+
+        :param IntegrationFinding finding: The finding data
+        :param int issue_id: The ID of the issue
+        :rtype: None
+        """
+
+        def _create_property():
+            """Create the property in a separate thread"""
+            if not finding.extra_data:
+                return
+            try:
+                Property(
+                    key="source_file_path",
+                    value=finding.extra_data.get("source_file_path"),
+                    parentId=issue_id,
+                    parentModule="issues",
+                ).create()
+            except Exception as exc:
+                # Log any errors that occur in the thread
+                logger.error(f"Error creating property for issue {issue_id}: {exc}")
+
+        # Start the property creation in a separate thread
+        thread = threading.Thread(target=_create_property, daemon=True)
+        thread.start()
+
     @staticmethod
     def get_consolidated_asset_identifier(
         finding: IntegrationFinding,
@@ -1731,7 +1811,7 @@ class ScannerIntegration(ABC):
         return None
 
     @staticmethod
-    def lookup_kev_and_upate_issue(
+    def lookup_kev_and_update_issue(
         cve: str, issue: regscale_models.Issue, cisa_kevs: Optional[ThreadSafeDict[str, Any]] = None
     ) -> regscale_models.Issue:
         """
@@ -1759,7 +1839,14 @@ class ScannerIntegration(ABC):
                 None,
             )
             if kev_data:
-                issue.dueDate = convert_datetime_to_regscale_string(datetime.strptime(kev_data["dueDate"], "%Y-%m-%d"))
+                # If kev due date is before the issue date created, add the difference to the date created
+                calculated_due_date = ScannerIntegration._calculate_kev_due_date(kev_data, issue.dateCreated)
+                if calculated_due_date:
+                    issue.dueDate = calculated_due_date
+                else:
+                    issue.dueDate = convert_datetime_to_regscale_string(
+                        datetime.strptime(kev_data["dueDate"], "%Y-%m-%d")
+                    )
                 issue.kevList = "Yes"
 
         return issue
@@ -2523,6 +2610,17 @@ class ScannerIntegration(ABC):
         issue.dateLastUpdated = get_current_datetime()
         issue.save()
 
+        if ScannerVariables.useMilestones:
+            regscale_models.Milestone(
+                title=f"Issue closed from {self.title} scan",
+                milestoneDate=issue.dateCompleted,
+                responsiblePersonId=self.assessor_id,
+                completed=True,
+                parentID=issue.id,
+                parentModule="issues",
+            ).create_or_update()
+            logger.debug("Created milestone for issue %s from %s tool", issue.id, self.title)
+
         with count_lock:
             self.closed_count += 1
             if issue.controlImplementationIds:
@@ -2600,6 +2698,9 @@ class ScannerIntegration(ABC):
         """
         # Do not close issues from other tools
         if issue.sourceReport != self.title:
+            logger.debug(
+                "Skipping issue %d from different source: %s (expected: %s)", issue.id, issue.sourceReport, self.title
+            )
             return False
 
         # If the issue has a vulnerability ID, check if it's still current for any asset
@@ -2611,14 +2712,19 @@ class ScannerIntegration(ABC):
 
             # Check if the issue's vulnerability is still current for any asset
             # If it is, we shouldn't close the issue
-            if any(
-                mapping.assetId in current_vulnerabilities
-                and issue.vulnerabilityId in current_vulnerabilities[mapping.assetId]
-                for mapping in vuln_mappings
-            ):
-                return False
+            for mapping in vuln_mappings:
+                if mapping.assetId in current_vulnerabilities:
+                    if issue.vulnerabilityId in current_vulnerabilities[mapping.assetId]:
+                        logger.debug(
+                            "Issue %d has current vulnerability %d for asset %d",
+                            issue.id,
+                            issue.vulnerabilityId,
+                            mapping.assetId,
+                        )
+                        return False
 
         # If we've checked all conditions and found no current vulnerabilities, we should close it
+        logger.debug("Issue %d has no current vulnerabilities, marking for closure", issue.id)
         return True
 
     @staticmethod
@@ -2901,6 +3007,65 @@ class ScannerIntegration(ABC):
             finding.first_seen = self.scan_date
         return existing_vuln
 
+    def _update_first_seen_date(
+        self, finding: IntegrationFinding, existing_vuln: Optional[regscale_models.VulnerabilityMapping]
+    ) -> None:
+        """
+        Update the first_seen date based on existing vulnerability mapping or scan date.
+
+        :param IntegrationFinding finding: The integration finding
+        :param Optional[regscale_models.VulnerabilityMapping] existing_vuln: The existing vulnerability mapping
+        :return: None
+        :rtype: None
+        """
+        if existing_vuln and existing_vuln.firstSeen:
+            finding.first_seen = existing_vuln.firstSeen
+        elif not finding.first_seen:
+            finding.first_seen = self.scan_date
+
+    def _update_date_created(self, finding: IntegrationFinding, issue: Optional[regscale_models.Issue]) -> None:
+        """
+        Update the date_created based on issue or scan date.
+
+        :param IntegrationFinding finding: The integration finding
+        :param Optional[regscale_models.Issue] issue: The existing issue
+        :return: None
+        :rtype: None
+        """
+        if issue and issue.dateFirstDetected:
+            finding.date_created = issue.dateFirstDetected
+        elif not finding.date_created:
+            finding.date_created = self.scan_date
+
+    def _update_due_date(self, finding: IntegrationFinding) -> None:
+        """
+        Update the due_date based on severity and configuration.
+
+        :param IntegrationFinding finding: The integration finding
+        :return: None
+        :rtype: None
+        """
+        finding.due_date = issue_due_date(
+            severity=finding.severity,
+            created_date=finding.date_created or self.scan_date,
+            title=self.title,
+            config=self.app.config,
+        )
+
+    def _update_last_seen_date(self, finding: IntegrationFinding) -> None:
+        """
+        Update the last_seen date if scan date is after first_seen.
+
+        :param IntegrationFinding finding: The integration finding
+        :return: None
+        :rtype: None
+        """
+        scan_date = date_obj(self.scan_date)
+        first_seen = date_obj(finding.first_seen)
+
+        if scan_date and first_seen and scan_date >= first_seen:
+            finding.last_seen = self.scan_date
+
     def update_finding_dates(
         self,
         finding: IntegrationFinding,
@@ -2916,28 +3081,17 @@ class ScannerIntegration(ABC):
         :return: The updated integration finding
         :rtype: IntegrationFinding
         """
-        if not finding.due_date:
-            if not existing_vuln:
-                finding.first_seen = self.scan_date
-                finding.date_created = self.scan_date
-                # From @mlongworth:
-                # It also appears that the suspense date (due date for remediation) is set based upon the import date rather
-                # than the scan date. This calculation needs to be based upon scan date e.g. scan date of 2/5/2024 severity
-                # High, should set the due date for remediation in the POA&M to 4/5/2024.
-                finding.due_date = issue_due_date(
-                    severity=finding.severity,
-                    created_date=finding.date_created or self.scan_date,
-                    title=self.title,
-                    config=self.app.config,
-                )
-            else:
-                finding.first_seen = existing_vuln.firstSeen if existing_vuln else finding.first_seen
-            if issue:
-                finding.date_created = issue.dateFirstDetected or finding.date_created
-            scan_date = date_obj(self.scan_date)
-            first_seen = date_obj(finding.first_seen)
-            if scan_date and first_seen and scan_date >= first_seen:
-                finding.last_seen = self.scan_date
+        if finding.due_date:
+            # If due_date is already set, only update last_seen if needed
+            self._update_last_seen_date(finding)
+            return finding
+
+        # Update dates for new findings
+        self._update_first_seen_date(finding, existing_vuln)
+        self._update_date_created(finding, issue)
+        self._update_due_date(finding)
+        self._update_last_seen_date(finding)
+
         return finding
 
     def update_scan(self, scan_history: regscale_models.ScanHistory) -> None:
@@ -2991,7 +3145,9 @@ class ScannerIntegration(ABC):
         :rtype: None
         """
         # Method is deprecated - using update_control_implementation_status_after_close instead
-        pass
+        logger.warning(
+            "update_control_implementation_status is deprecated - using update_control_implementation_status_after_close instead"
+        )
 
     def update_regscale_checklists(self, findings: List[IntegrationFinding]) -> int:
         """
@@ -3074,3 +3230,31 @@ class ScannerIntegration(ABC):
             impact=finding.impact,
             recommendationForMitigation=finding.recommendation_for_mitigation,
         ).create()
+
+    @staticmethod
+    def _calculate_kev_due_date(kev_data: dict, issue_date_created: str) -> Optional[str]:
+        """
+        Calculate the due date for a KEV issue based on the difference between
+        KEV due date and date added, then add that difference to the issue creation date.
+
+        :param dict kev_data: KEV data containing dueDate and dateAdded
+        :param str issue_date_created: The issue creation date string
+        :return: Calculated due date as a RegScale formatted string or None
+        :rtype: Optional[str]
+        """
+        from datetime import datetime
+
+        from regscale.core.app.utils.app_utils import convert_datetime_to_regscale_string
+
+        if datetime.strptime(kev_data["dueDate"], "%Y-%m-%d") < datetime.strptime(
+            issue_date_created, "%Y-%m-%d %H:%M:%S"
+        ):
+            # diff kev due date and kev dateAdded
+            diff = datetime.strptime(kev_data["dueDate"], "%Y-%m-%d") - datetime.strptime(
+                kev_data["dateAdded"], "%Y-%m-%d"
+            )
+            # add diff to issue.dateCreated
+            return convert_datetime_to_regscale_string(
+                datetime.strptime(issue_date_created, "%Y-%m-%d %H:%M:%S") + diff
+            )
+        return None

regscale/integrations/variables.py

@@ -26,3 +26,4 @@ class ScannerVariables(metaclass=RsVariablesMeta):
     maxRetries: RsVariableType(int, "3", default=3, required=False)  # type: ignore
     timeout: RsVariableType(int, "60", default=60, required=False)  # type: ignore
     complianceCreation: RsVariableType(str, "Assessment|Issue|POAM", default="Assessment", required=False)  # type: ignore # noqa: F722,F821
+    useMilestones: RsVariableType(bool, "true|false", default=False, required=False)  # type: ignore # noqa: F722,F722,F821

regscale/models/integration_models/amazon_models/inspector_scan.py

@@ -3,18 +3,17 @@ AWS Inspector Scan information
 """
 
 import re
-from typing import List, Optional, Tuple
-
 from pathlib import Path
+from typing import List, Optional, Tuple
 
 from regscale.core.app.application import Application
 from regscale.core.app.logz import create_logger
-from regscale.core.app.utils.app_utils import get_current_datetime, is_valid_fqdn
+from regscale.core.app.utils.app_utils import is_valid_fqdn
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
 from regscale.models import ImportValidater
 from regscale.models.integration_models.amazon_models.inspector import InspectorRecord
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
-from regscale.models.regscale_models.asset import Asset
-from regscale.models.regscale_models.vulnerability import Vulnerability
+from regscale.models.regscale_models import Asset, AssetStatus, IssueStatus, Vulnerability
 
 
 class InspectorScan(FlatFileImporter):
@@ -101,26 +100,17 @@ class InspectorScan(FlatFileImporter):
         # Container Image, Virtual Machine (VM), etc.
         asset_type = self.amazon_type_map().get(dat.resource_type, "Other")
 
-        return Asset(
-            **{
-                "id": 0,
-                "name": hostname,
-                "awsIdentifier": hostname,
-                "ipAddress": "",
-                "isPublic": True,
-                "status": "Active (On Network)",
-                "assetCategory": "Hardware",
-                "bLatestScan": True,
-                "bAuthenticatedScan": True,
-                "scanningTool": self.name,
-                "assetOwnerId": self.config["userId"],
-                "assetType": asset_type,
-                "fqdn": hostname if is_valid_fqdn(hostname) else None,
-                "operatingSystem": Asset.find_os(distro),
-                "systemAdministratorId": self.config["userId"],
-                "parentId": self.attributes.parent_id,
-                "parentModule": self.attributes.parent_module,
-            }
+        return IntegrationAsset(
+            identifier=hostname,
+            name=hostname,
+            ip_address="0.0.0.0",
+            cpu=0,
+            ram=0,
+            status=AssetStatus.Active.value,
+            asset_type=asset_type,
+            asset_category="Software",
+            operating_system=Asset.find_os(distro),
+            fqdn=hostname if is_valid_fqdn(hostname) else None,
         )
 
     def create_vuln(self, dat: Optional[InspectorRecord] = None, **kwargs) -> Optional[Vulnerability]:
@@ -132,44 +122,29 @@ class InspectorScan(FlatFileImporter):
         :rtype: Optional[Vulnerability]
         """
         hostname = dat.resource_id
-        distro = dat.platform
         cve: str = dat.vulnerability_id
         description: str = dat.description
         title = dat.title if dat.title else dat.description
         aws_severity = dat.severity
         severity = self.severity_mapper(aws_severity)
-        config = self.attributes.app.config
-        asset_match = [asset for asset in self.data["assets"] if asset.name == hostname]
-        asset = asset_match[0] if asset_match else None
-        if dat and asset_match:
-            return Vulnerability(
-                id=0,
-                scanId=0,  # set later
-                parentId=asset.id,
-                parentModule="assets",
-                ipAddress="0.0.0.0",  # No ip address available
-                lastSeen=dat.last_seen,
-                firstSeen=dat.first_seen,
-                daysOpen=None,
-                dns=hostname,
-                mitigated=None,
-                operatingSystem=(Asset.find_os(distro) if Asset.find_os(distro) else None),
-                severity=severity,
-                plugInName=dat.title,
-                plugInId=self.convert_cve_string_to_int(dat.vulnerability_id),
-                cve=cve,
-                vprScore=None,
-                tenantsId=0,
-                title=f"{description} on asset {asset.name}",
+        if dat:
+            return IntegrationFinding(
+                title=title,
                 description=description,
-                plugInText=title,
-                createdById=config["userId"],
-                lastUpdatedById=config["userId"],
-                dateCreated=get_current_datetime(),
-                extra_data={
-                    "solution": dat.remediation,
-                    "proof": dat.finding_arn,
-                },
+                severity=self.determine_severity(severity),
+                status=IssueStatus.Open.value,
+                ip_address="0.0.0.0",
+                plugin_text=title,
+                plugin_name=dat.title,
+                plugin_id=self.convert_cve_string_to_int(dat.vulnerability_id),
+                asset_identifier=hostname,
+                remediation=dat.remediation,
+                cve=cve,
+                first_seen=dat.first_seen,
+                last_seen=dat.last_seen,
+                scan_date=self.scan_date,
+                category="Software",
+                control_labels=[],
             )
         return None