regscale-cli 6.20.10.0__py3-none-any.whl → 6.21.1.0__py3-none-any.whl

regscale/integrations/integration_override.py

@@ -2,8 +2,6 @@
 A simple singleton class that loads custom integration mappings, if available
 """
 
-from regscale.core.app.application import Application
-
 # pylint: disable=C0415
 
 
@@ -74,9 +72,12 @@ class IntegrationOverride:
         :return: The mapped field name
         :rtype: Optional[str]
         """
-        if integration and self.mapping_exists(integration, field_name):
+        if integration and field_name and self.mapping_exists(integration, field_name):
             integration_map = self.mapping.get(integration.lower(), {})
-            return integration_map.get(field_name.lower())
+            # Find the actual key that matches case-insensitively
+            for key in integration_map.keys():
+                if key.lower() == field_name.lower():
+                    return integration_map.get(key)
         return None
 
     def mapping_exists(self, integration: str, field_name: str) -> bool:
@@ -88,8 +89,16 @@ class IntegrationOverride:
         :return: Whether the mapping exists
         :rtype: bool
         """
+        if not integration or not field_name:
+            return False
         the_map = self.mapping.get(integration.lower())
-        return the_map and field_name.lower() in the_map and the_map.get(field_name.lower()) != "default"
+        if not the_map:
+            return False
+        # Find the actual key that matches case-insensitively
+        for key in the_map.keys():
+            if key.lower() == field_name.lower():
+                return the_map.get(key) != "default"
+        return False
 
     def field_map_validation(self, obj: Any, model_type: str) -> Optional[str]:
         """
@@ -131,7 +140,7 @@ class IntegrationOverride:
                 },
             }
             # The type an associated fields we are able to override. Limited for now.
-            supported_fields = {"asset": {"ipAddress", "name", "fqdn", "dns"}}
+            supported_fields = {"asset": {"ipAddress", "name", "fqdn", "dns"}, "issue": {"dateFirstDetected"}}
             if regscale_field not in supported_fields.get(model_type.lower(), set()):
                 return match
 

regscale/integrations/public/fedramp/fedramp_five.py

@@ -1764,7 +1764,7 @@ def update_existing_control(
 
     # Convert the model to a dict and back to a model to workaround these odd 400 errors.
     try:
-        ControlImplementation(**control.dict()).save()
+        control.save()
     except Exception as e:
         logger.warning(f"Error updating control: {control.id} - {e}")
 

regscale/integrations/public/fedramp/markdown_parser.py

@@ -7,6 +7,7 @@ import re
 import logging
 import zipfile  # Assuming you need this for other file handling
 import pypandoc
+import re
 from collections import defaultdict
 from typing import Dict, TextIO, Optional, Tuple
 from regscale.models import ProfileMapping
@@ -108,7 +109,12 @@ class MDDocParser:
         """
         # Extract control ID and clean it
         html_free_line = self.clean_html_and_newlines(line)
-        clean_line = html_free_line.replace(FULL_SUMMARY_TOKEN, "")
+        # Use regex to find "what" case-insensitively and split
+        pattern = re.compile(r"what", re.IGNORECASE)
+        if pattern.search(html_free_line):
+            clean_line = pattern.split(html_free_line)[0].strip()
+        else:
+            clean_line = html_free_line
         if not clean_line:
             return None
         clean_control_id_from_line = clean_line.strip()

regscale/integrations/scanner_integration.py

@@ -28,7 +28,7 @@ from regscale.integrations.commercial.durosuite.variables import DuroSuiteVariab
 from regscale.integrations.commercial.stig_mapper_integration.mapping_engine import StigMappingEngine as STIGMapper
 from regscale.integrations.public.cisa import pull_cisa_kev
 from regscale.integrations.variables import ScannerVariables
-from regscale.models import DateTimeEncoder, OpenIssueDict, regscale_models
+from regscale.models import DateTimeEncoder, OpenIssueDict, Property, regscale_models
 from regscale.utils.threading import ThreadSafeDict, ThreadSafeList
 
 logger = logging.getLogger(__name__)
@@ -407,7 +407,7 @@ class IntegrationFinding:
     issue_type: str = "Risk"
     date_created: str = dataclasses.field(default_factory=get_current_datetime)
     date_last_updated: str = dataclasses.field(default_factory=get_current_datetime)
-    due_date: str = dataclasses.field(default_factory=lambda: date_str(days_from_today(60)))
+    due_date: str = ""  # dataclasses.field(default_factory=lambda: date_str(days_from_today(60)))
     external_id: str = ""
     gaps: str = ""
     observations: str = ""
@@ -423,6 +423,9 @@ class IntegrationFinding:
     planned_milestone_changes: Optional[str] = None
     adjusted_risk_rating: Optional[str] = None
     risk_adjustment: str = "No"
+
+    # Compliance fields
+    assessment_id: Optional[int] = None
     operational_requirements: Optional[str] = None
     deviation_rationale: Optional[str] = None
     is_cwe: bool = False
@@ -456,13 +459,16 @@ class IntegrationFinding:
     source_rule_id: Optional[str] = None
     vulnerability_type: Optional[str] = None
 
-    # CoalFre POAM
+    # CoalFire POAM
     basis_for_adjustment: Optional[str] = None
     poam_id: Optional[str] = None
 
     # Additional fields from Wiz integration
     vpr_score: Optional[float] = None
 
+    # Extra data field for miscellaneous data
+    extra_data: Dict[str, Any] = dataclasses.field(default_factory=dict)
+
     def __post_init__(self):
         """Validate and adjust types after initialization."""
         # Set default date values if empty
@@ -1609,6 +1615,23 @@ class ScannerIntegration(ABC):
         issue.securityPlanId = self.plan_id if not self.is_component else None
         issue.identification = finding.identification
         issue.dateFirstDetected = finding.first_seen
+        # Ensure a due date is always set using configured policy defaults (e.g., FedRAMP)
+        if not finding.due_date:
+            try:
+                base_created = finding.date_created or issue.dateCreated
+                finding.due_date = issue_due_date(
+                    severity=finding.severity,
+                    created_date=base_created,
+                    title=self.title,
+                )
+            except Exception:
+                # Final fallback to a Low severity default if anything goes wrong
+                base_created = finding.date_created or issue.dateCreated
+                finding.due_date = issue_due_date(
+                    severity=regscale_models.IssueSeverity.Low,
+                    created_date=base_created,
+                    title=self.title,
+                )
         issue.dueDate = finding.due_date
         issue.description = description
         issue.sourceReport = finding.source_report or self.title
@@ -1619,15 +1642,21 @@ class ScannerIntegration(ABC):
         issue.integrationFindingId = self.get_finding_identifier(finding)
         issue.poamComments = finding.poam_comments
         issue.cve = finding.cve
+        issue.assessmentId = finding.assessment_id
         control_id = self.get_control_implementation_id_for_cci(finding.cci_ref) if finding.cci_ref else None
         issue.controlId = control_id  # TODO REMOVE
         # Add the control implementation ids and the cci ref if it exists
         # Get control implementation ID for CCI if it exists
         # Only add CCI control ID if it exists
         cci_control_ids = [control_id] if control_id is not None else []
-        issue.affectedControls = finding.affected_controls
+        # Ensure failed control labels (e.g., AC-4(21)) are present in affectedControls
+        if finding.affected_controls:
+            issue.affectedControls = finding.affected_controls
+        elif finding.control_labels:
+            issue.affectedControls = ", ".join(sorted({cl for cl in finding.control_labels if cl}))
 
         issue.controlImplementationIds = list(set(finding._control_implementation_ids + cci_control_ids))  # noqa
+        # Always ensure isPoam reflects current settings, even when updating existing issues
         issue.isPoam = is_poam
         issue.basisForAdjustment = (
             finding.basis_for_adjustment if finding.basis_for_adjustment else f"{self.title} import"
@@ -1644,17 +1673,22 @@ class ScannerIntegration(ABC):
         issue.operationalRequirement = finding.operational_requirements
         issue.deviationRationale = finding.deviation_rationale
         issue.dateLastUpdated = get_current_datetime()
+        ## set affected controls if they exist
+        issue.affectedControls = finding.affected_controls
 
         if finding.cve:
-            issue = self.lookup_kev_and_upate_issue(cve=finding.cve, issue=issue, cisa_kevs=self._kev_data)
+            issue = self.lookup_kev_and_update_issue(cve=finding.cve, issue=issue, cisa_kevs=self._kev_data)
 
         if existing_issue:
-            logger.debug("Saving existing issue %s with assetIdentifier: %s", issue.id, issue.assetIdentifier)
+            logger.debug("Saving Old Issue: %s  with assetIdentifier: %s", issue.id, issue.assetIdentifier)
             issue.save(bulk=True)
+            logger.debug("Saved existing issue %s with assetIdentifier: %s", issue.id, issue.assetIdentifier)
+
         else:
             issue = issue.create_or_update(
                 bulk_update=True, defaults={"otherIdentifier": self._get_other_identifier(finding, is_poam)}
             )
+            self.extra_data_to_properties(finding, issue.id)
 
         self._handle_property_and_milestone_creation(issue, finding, existing_issue)
         return issue
@@ -1730,6 +1764,35 @@ class ScannerIntegration(ABC):
             else:
                 logger.debug("No milestone created for issue %s from finding %s", issue.id, finding.external_id)
 
+    @staticmethod
+    def extra_data_to_properties(finding: IntegrationFinding, issue_id: int) -> None:
+        """
+        Adds extra data to properties for an issue in a separate thread
+
+        :param IntegrationFinding finding: The finding data
+        :param int issue_id: The ID of the issue
+        :rtype: None
+        """
+
+        def _create_property():
+            """Create the property in a separate thread"""
+            if not finding.extra_data:
+                return
+            try:
+                Property(
+                    key="source_file_path",
+                    value=finding.extra_data.get("source_file_path"),
+                    parentId=issue_id,
+                    parentModule="issues",
+                ).create()
+            except Exception as exc:
+                # Log any errors that occur in the thread
+                logger.error(f"Error creating property for issue {issue_id}: {exc}")
+
+        # Start the property creation in a separate thread
+        thread = threading.Thread(target=_create_property, daemon=True)
+        thread.start()
+
     @staticmethod
     def get_consolidated_asset_identifier(
         finding: IntegrationFinding,
@@ -1776,7 +1839,7 @@ class ScannerIntegration(ABC):
         return None
 
     @staticmethod
-    def lookup_kev_and_upate_issue(
+    def lookup_kev_and_update_issue(
         cve: str, issue: regscale_models.Issue, cisa_kevs: Optional[ThreadSafeDict[str, Any]] = None
     ) -> regscale_models.Issue:
         """
@@ -1804,7 +1867,14 @@ class ScannerIntegration(ABC):
                 None,
             )
             if kev_data:
-                issue.dueDate = convert_datetime_to_regscale_string(datetime.strptime(kev_data["dueDate"], "%Y-%m-%d"))
+                # If kev due date is before the issue date created, add the difference to the date created
+                calculated_due_date = ScannerIntegration._calculate_kev_due_date(kev_data, issue.dateCreated)
+                if calculated_due_date:
+                    issue.dueDate = calculated_due_date
+                else:
+                    issue.dueDate = convert_datetime_to_regscale_string(
+                        datetime.strptime(kev_data["dueDate"], "%Y-%m-%d")
+                    )
                 issue.kevList = "Yes"
 
         return issue
@@ -2656,6 +2726,9 @@ class ScannerIntegration(ABC):
         """
         # Do not close issues from other tools
         if issue.sourceReport != self.title:
+            logger.debug(
+                "Skipping issue %d from different source: %s (expected: %s)", issue.id, issue.sourceReport, self.title
+            )
             return False
 
         # If the issue has a vulnerability ID, check if it's still current for any asset
@@ -2667,14 +2740,19 @@ class ScannerIntegration(ABC):
 
             # Check if the issue's vulnerability is still current for any asset
             # If it is, we shouldn't close the issue
-            if any(
-                mapping.assetId in current_vulnerabilities
-                and issue.vulnerabilityId in current_vulnerabilities[mapping.assetId]
-                for mapping in vuln_mappings
-            ):
-                return False
+            for mapping in vuln_mappings:
+                if mapping.assetId in current_vulnerabilities:
+                    if issue.vulnerabilityId in current_vulnerabilities[mapping.assetId]:
+                        logger.debug(
+                            "Issue %d has current vulnerability %d for asset %d",
+                            issue.id,
+                            issue.vulnerabilityId,
+                            mapping.assetId,
+                        )
+                        return False
 
         # If we've checked all conditions and found no current vulnerabilities, we should close it
+        logger.debug("Issue %d has no current vulnerabilities, marking for closure", issue.id)
         return True
 
     @staticmethod
@@ -2957,6 +3035,65 @@ class ScannerIntegration(ABC):
             finding.first_seen = self.scan_date
         return existing_vuln
 
+    def _update_first_seen_date(
+        self, finding: IntegrationFinding, existing_vuln: Optional[regscale_models.VulnerabilityMapping]
+    ) -> None:
+        """
+        Update the first_seen date based on existing vulnerability mapping or scan date.
+
+        :param IntegrationFinding finding: The integration finding
+        :param Optional[regscale_models.VulnerabilityMapping] existing_vuln: The existing vulnerability mapping
+        :return: None
+        :rtype: None
+        """
+        if existing_vuln and existing_vuln.firstSeen:
+            finding.first_seen = existing_vuln.firstSeen
+        elif not finding.first_seen:
+            finding.first_seen = self.scan_date
+
+    def _update_date_created(self, finding: IntegrationFinding, issue: Optional[regscale_models.Issue]) -> None:
+        """
+        Update the date_created based on issue or scan date.
+
+        :param IntegrationFinding finding: The integration finding
+        :param Optional[regscale_models.Issue] issue: The existing issue
+        :return: None
+        :rtype: None
+        """
+        if issue and issue.dateFirstDetected:
+            finding.date_created = issue.dateFirstDetected
+        elif not finding.date_created:
+            finding.date_created = self.scan_date
+
+    def _update_due_date(self, finding: IntegrationFinding) -> None:
+        """
+        Update the due_date based on severity and configuration.
+
+        :param IntegrationFinding finding: The integration finding
+        :return: None
+        :rtype: None
+        """
+        finding.due_date = issue_due_date(
+            severity=finding.severity,
+            created_date=finding.date_created or self.scan_date,
+            title=self.title,
+            config=self.app.config,
+        )
+
+    def _update_last_seen_date(self, finding: IntegrationFinding) -> None:
+        """
+        Update the last_seen date if scan date is after first_seen.
+
+        :param IntegrationFinding finding: The integration finding
+        :return: None
+        :rtype: None
+        """
+        scan_date = date_obj(self.scan_date)
+        first_seen = date_obj(finding.first_seen)
+
+        if scan_date and first_seen and scan_date >= first_seen:
+            finding.last_seen = self.scan_date
+
     def update_finding_dates(
         self,
         finding: IntegrationFinding,
@@ -2972,28 +3109,17 @@ class ScannerIntegration(ABC):
         :return: The updated integration finding
         :rtype: IntegrationFinding
         """
-        if not finding.due_date:
-            if not existing_vuln:
-                finding.first_seen = self.scan_date
-                finding.date_created = self.scan_date
-                # From @mlongworth:
-                # It also appears that the suspense date (due date for remediation) is set based upon the import date rather
-                # than the scan date. This calculation needs to be based upon scan date e.g. scan date of 2/5/2024 severity
-                # High, should set the due date for remediation in the POA&M to 4/5/2024.
-                finding.due_date = issue_due_date(
-                    severity=finding.severity,
-                    created_date=finding.date_created or self.scan_date,
-                    title=self.title,
-                    config=self.app.config,
-                )
-            else:
-                finding.first_seen = existing_vuln.firstSeen if existing_vuln else finding.first_seen
-            if issue:
-                finding.date_created = issue.dateFirstDetected or finding.date_created
-            scan_date = date_obj(self.scan_date)
-            first_seen = date_obj(finding.first_seen)
-            if scan_date and first_seen and scan_date >= first_seen:
-                finding.last_seen = self.scan_date
+        if finding.due_date:
+            # If due_date is already set, only update last_seen if needed
+            self._update_last_seen_date(finding)
+            return finding
+
+        # Update dates for new findings
+        self._update_first_seen_date(finding, existing_vuln)
+        self._update_date_created(finding, issue)
+        self._update_due_date(finding)
+        self._update_last_seen_date(finding)
+
         return finding
 
     def update_scan(self, scan_history: regscale_models.ScanHistory) -> None:
@@ -3047,7 +3173,9 @@ class ScannerIntegration(ABC):
         :rtype: None
         """
         # Method is deprecated - using update_control_implementation_status_after_close instead
-        pass
+        logger.warning(
+            "update_control_implementation_status is deprecated - using update_control_implementation_status_after_close instead"
+        )
 
     def update_regscale_checklists(self, findings: List[IntegrationFinding]) -> int:
         """
@@ -3130,3 +3258,31 @@ class ScannerIntegration(ABC):
             impact=finding.impact,
             recommendationForMitigation=finding.recommendation_for_mitigation,
         ).create()
+
+    @staticmethod
+    def _calculate_kev_due_date(kev_data: dict, issue_date_created: str) -> Optional[str]:
+        """
+        Calculate the due date for a KEV issue based on the difference between
+        KEV due date and date added, then add that difference to the issue creation date.
+
+        :param dict kev_data: KEV data containing dueDate and dateAdded
+        :param str issue_date_created: The issue creation date string
+        :return: Calculated due date as a RegScale formatted string or None
+        :rtype: Optional[str]
+        """
+        from datetime import datetime
+
+        from regscale.core.app.utils.app_utils import convert_datetime_to_regscale_string
+
+        if datetime.strptime(kev_data["dueDate"], "%Y-%m-%d") < datetime.strptime(
+            issue_date_created, "%Y-%m-%d %H:%M:%S"
+        ):
+            # diff kev due date and kev dateAdded
+            diff = datetime.strptime(kev_data["dueDate"], "%Y-%m-%d") - datetime.strptime(
+                kev_data["dateAdded"], "%Y-%m-%d"
+            )
+            # add diff to issue.dateCreated
+            return convert_datetime_to_regscale_string(
+                datetime.strptime(issue_date_created, "%Y-%m-%d %H:%M:%S") + diff
+            )
+        return None

regscale/models/app_models/__init__.py

@@ -3,3 +3,4 @@
 from .mapping import Mapping
 from .import_validater import ImportValidater
 from .datetime_encoder import DateTimeEncoder
+from .click import NotRequiredIf, regscale_id, regscale_module

regscale/models/integration_models/amazon_models/inspector_scan.py

@@ -3,18 +3,17 @@ AWS Inspector Scan information
 """
 
 import re
-from typing import List, Optional, Tuple
-
 from pathlib import Path
+from typing import List, Optional, Tuple
 
 from regscale.core.app.application import Application
 from regscale.core.app.logz import create_logger
-from regscale.core.app.utils.app_utils import get_current_datetime, is_valid_fqdn
+from regscale.core.app.utils.app_utils import is_valid_fqdn
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding
 from regscale.models import ImportValidater
 from regscale.models.integration_models.amazon_models.inspector import InspectorRecord
 from regscale.models.integration_models.flat_file_importer import FlatFileImporter
-from regscale.models.regscale_models.asset import Asset
-from regscale.models.regscale_models.vulnerability import Vulnerability
+from regscale.models.regscale_models import Asset, AssetStatus, IssueStatus, Vulnerability
 
 
 class InspectorScan(FlatFileImporter):
@@ -101,26 +100,17 @@ class InspectorScan(FlatFileImporter):
         # Container Image, Virtual Machine (VM), etc.
         asset_type = self.amazon_type_map().get(dat.resource_type, "Other")
 
-        return Asset(
-            **{
-                "id": 0,
-                "name": hostname,
-                "awsIdentifier": hostname,
-                "ipAddress": "",
-                "isPublic": True,
-                "status": "Active (On Network)",
-                "assetCategory": "Hardware",
-                "bLatestScan": True,
-                "bAuthenticatedScan": True,
-                "scanningTool": self.name,
-                "assetOwnerId": self.config["userId"],
-                "assetType": asset_type,
-                "fqdn": hostname if is_valid_fqdn(hostname) else None,
-                "operatingSystem": Asset.find_os(distro),
-                "systemAdministratorId": self.config["userId"],
-                "parentId": self.attributes.parent_id,
-                "parentModule": self.attributes.parent_module,
-            }
+        return IntegrationAsset(
+            identifier=hostname,
+            name=hostname,
+            ip_address="0.0.0.0",
+            cpu=0,
+            ram=0,
+            status=AssetStatus.Active.value,
+            asset_type=asset_type,
+            asset_category="Software",
+            operating_system=Asset.find_os(distro),
+            fqdn=hostname if is_valid_fqdn(hostname) else None,
         )
 
     def create_vuln(self, dat: Optional[InspectorRecord] = None, **kwargs) -> Optional[Vulnerability]:
@@ -132,44 +122,29 @@ class InspectorScan(FlatFileImporter):
         :rtype: Optional[Vulnerability]
         """
         hostname = dat.resource_id
-        distro = dat.platform
         cve: str = dat.vulnerability_id
         description: str = dat.description
         title = dat.title if dat.title else dat.description
         aws_severity = dat.severity
         severity = self.severity_mapper(aws_severity)
-        config = self.attributes.app.config
-        asset_match = [asset for asset in self.data["assets"] if asset.name == hostname]
-        asset = asset_match[0] if asset_match else None
-        if dat and asset_match:
-            return Vulnerability(
-                id=0,
-                scanId=0,  # set later
-                parentId=asset.id,
-                parentModule="assets",
-                ipAddress="0.0.0.0",  # No ip address available
-                lastSeen=dat.last_seen,
-                firstSeen=dat.first_seen,
-                daysOpen=None,
-                dns=hostname,
-                mitigated=None,
-                operatingSystem=(Asset.find_os(distro) if Asset.find_os(distro) else None),
-                severity=severity,
-                plugInName=dat.title,
-                plugInId=self.convert_cve_string_to_int(dat.vulnerability_id),
-                cve=cve,
-                vprScore=None,
-                tenantsId=0,
-                title=f"{description} on asset {asset.name}",
+        if dat:
+            return IntegrationFinding(
+                title=title,
                 description=description,
-                plugInText=title,
-                createdById=config["userId"],
-                lastUpdatedById=config["userId"],
-                dateCreated=get_current_datetime(),
-                extra_data={
-                    "solution": dat.remediation,
-                    "proof": dat.finding_arn,
-                },
+                severity=self.determine_severity(severity),
+                status=IssueStatus.Open.value,
+                ip_address="0.0.0.0",
+                plugin_text=title,
+                plugin_name=dat.title,
+                plugin_id=self.convert_cve_string_to_int(dat.vulnerability_id),
+                asset_identifier=hostname,
+                remediation=dat.remediation,
+                cve=cve,
+                first_seen=dat.first_seen,
+                last_seen=dat.last_seen,
+                scan_date=self.scan_date,
+                category="Software",
+                control_labels=[],
             )
         return None