regscale-cli 6.23.0.1__py3-none-any.whl → 6.24.0.1__py3-none-any.whl

regscale/integrations/scanner_integration.py

@@ -22,10 +22,11 @@ from regscale.core.app.application import Application
 from regscale.core.app.utils.api_handler import APIHandler
 from regscale.core.app.utils.app_utils import create_progress_object, get_current_datetime
 from regscale.core.app.utils.catalog_utils.common import objective_to_control_dot
-from regscale.core.utils.date import date_obj, date_str, datetime_str, days_from_today, get_day_increment
+from regscale.core.utils.date import date_obj, date_str, datetime_str, get_day_increment
 from regscale.integrations.commercial.durosuite.process_devices import scan_durosuite_devices
 from regscale.integrations.commercial.durosuite.variables import DuroSuiteVariables
 from regscale.integrations.commercial.stig_mapper_integration.mapping_engine import StigMappingEngine as STIGMapper
+from regscale.integrations.due_date_handler import DueDateHandler
 from regscale.integrations.public.cisa import pull_cisa_kev
 from regscale.integrations.variables import ScannerVariables
 from regscale.models import DateTimeEncoder, OpenIssueDict, Property, regscale_models
@@ -322,6 +323,8 @@ class IntegrationFinding:
     :param str impact: The impact of the finding, defaults to an empty string.
     :param str recommendation_for_mitigation: Recommendations for mitigating the finding, defaults to an empty string.
     :param str asset_identifier: The identifier of the asset associated with the finding, defaults to an empty string.
+    :param str issue_asset_identifier_value: This is the value of all the assets affected by the issue, defaults to an
+    empty string.
     :param Optional[str] cci_ref: The Common Configuration Enumeration reference for the finding, defaults to None.
     :param str rule_id: The rule ID of the finding, defaults to an empty string.
     :param str rule_version: The version of the rule associated with the finding, defaults to an empty string.
@@ -418,6 +421,7 @@ class IntegrationFinding:
     impact: str = ""
     recommendation_for_mitigation: str = ""
     asset_identifier: str = ""
+    issue_asset_identifier_value: Optional[str] = None
     comments: Optional[str] = None
     source_report: Optional[str] = None
     point_of_contact: Optional[str] = None
@@ -657,6 +661,10 @@ class ScannerIntegration(ABC):
 
         # Set configuration options from kwargs
         self.suppress_asset_not_found_errors = kwargs.get("suppress_asset_not_found_errors", False)
+
+        # Initialize due date handler for this integration
+        self.due_date_handler = DueDateHandler(self.title, config=self.app.config)
+
         if self.is_component:
             self.component = regscale_models.Component.get_object(self.plan_id)
             self.parent_module: str = regscale_models.Component.get_module_string()
@@ -752,6 +760,74 @@ class ScannerIntegration(ABC):
 
         return regscale_models.Issue.get_user_id()
 
+    def get_user_organization_id(self, user_id: Optional[str]) -> Optional[int]:
+        """
+        Get the organization ID for a user.
+
+        :param Optional[str] user_id: The user ID to look up
+        :return: The organization ID or None if not found
+        :rtype: Optional[int]
+        """
+        if not user_id:
+            return None
+
+        try:
+            from regscale.models import User
+
+            user = User.get_object(user_id)
+            return user.orgId if user else None
+        except Exception as e:
+            logger.debug(f"Unable to get user organization for user {user_id}: {e}")
+            return None
+
+    def get_ssp_organization_id(self) -> Optional[int]:
+        """
+        Get the organization ID from the security plan.
+
+        :return: The organization ID or None if not found
+        :rtype: Optional[int]
+        """
+        try:
+            from regscale.models import SecurityPlan
+
+            if ssp := SecurityPlan.get_object(self.plan_id):
+                # First try to get organization from SSP owner
+                if getattr(ssp, "systemOwnerId"):
+                    if owner_org_id := self.get_user_organization_id(ssp.systemOwnerId):
+                        return owner_org_id
+                # Fallback to SSP's direct organization
+                return ssp.orgId
+        except Exception as e:
+            logger.debug(f"Unable to get SSP organization for plan {self.plan_id}: {e}")
+
+        return None
+
+    def determine_issue_organization_id(self, issue_owner_id: Optional[str]) -> Optional[int]:
+        """
+        Determine the organization ID for an issue based on the expected behavior:
+
+        1. If Issue Owner is set and has an Org, use Issue Owner's Org
+        2. Else if SSP Owner has an Org, use SSP Owner's Org
+        3. Else use SSP's Org if set
+
+        :param Optional[str] issue_owner_id: The issue owner ID
+        :return: The organization ID or None
+        :rtype: Optional[int]
+        """
+        # First check if issue owner has an organization
+        if issue_owner_id:
+            if owner_org_id := self.get_user_organization_id(issue_owner_id):
+                logger.debug(f"Setting issue organization {owner_org_id} from issue owner {issue_owner_id}")
+                return owner_org_id
+
+        # Fallback to SSP organization (which includes SSP owner check)
+        if ssp_org_id := self.get_ssp_organization_id():
+            logger.debug(f"Setting issue organization {ssp_org_id} from SSP {self.plan_id}")
+            return ssp_org_id
+
+        logger.debug(f"No organization found for issue owner {issue_owner_id} or SSP {self.plan_id}")
+        return None
+
     def get_cci_to_control_map(self) -> ThreadSafeDict[str, set[int]] | dict:
         """
         Gets the CCI to control map
@@ -1046,39 +1122,104 @@ class ScannerIntegration(ABC):
         :param Optional[str] component_name: The name of the component to associate the asset with. If None, the asset
                                           is added directly to the security plan without a component association.
         """
-        # Continue with normal asset creation/update
         if not asset.identifier:
             logger.warning("Asset has no identifier, skipping")
             return
 
-        component = getattr(self, "component") if self.is_component else None
-        if component_name:
-            logger.debug("Searching for component: %s...", component_name)
-            component = component or self.components_by_title.get(component_name)
-            if not component:
-                logger.debug("No existing component found with name %s, proceeding to create it...", component_name)
-                component = regscale_models.Component(
-                    title=component_name,
-                    componentType=asset.component_type,
-                    securityPlansId=self.plan_id,
-                    description=component_name,
-                    componentOwnerId=self.get_assessor_id(),
-                ).get_or_create()
-                self.components.append(component)
-            if component.securityPlansId and not self.is_component:
-                component_mapping = regscale_models.ComponentMapping(
-                    componentId=component.id,
-                    securityPlanId=self.plan_id,
-                )
-                component_mapping.get_or_create()
-            self.components_by_title[component_name] = component
+        # Get or create component if needed
+        component = self._get_or_create_component_for_asset(asset, component_name)
 
+        # Create or update the asset
         created, existing_or_new_asset = self.create_new_asset(asset, component=None)
 
-        # update results expects a dict[str, list] to update result counts
+        # Update result counts
         self.update_result_counts("assets", {"created": [1] if created else [], "updated": [] if created else [1]})
 
-        # If the asset is associated with a component, create a mapping between them.
+        # Handle component mapping and DuroSuite processing
+        self._handle_component_mapping_and_durosuite(existing_or_new_asset, component, asset, created)
+
+    def _get_or_create_component_for_asset(
+        self, asset: IntegrationAsset, component_name: Optional[str]
+    ) -> Optional[regscale_models.Component]:
+        """
+        Get or create a component for the asset if component_name is provided.
+
+        :param IntegrationAsset asset: The asset being processed
+        :param Optional[str] component_name: Name of the component to associate with
+        :return: The component object or None
+        :rtype: Optional[regscale_models.Component]
+        """
+        if not component_name:
+            return getattr(self, "component") if self.is_component else None
+
+        component = getattr(self, "component") if self.is_component else None
+        component = component or self.components_by_title.get(component_name)
+
+        if not component:
+            component = self._create_new_component(asset, component_name)
+
+        self._handle_component_mapping(component)
+        self.components_by_title[component_name] = component
+        return component
+
+    def _create_new_component(self, asset: IntegrationAsset, component_name: str) -> regscale_models.Component:
+        """
+        Create a new component for the asset.
+
+        :param IntegrationAsset asset: The asset being processed
+        :param str component_name: Name of the component to create
+        :return: The newly created component
+        :rtype: regscale_models.Component
+        """
+        logger.debug("No existing component found with name %s, proceeding to create it...", component_name)
+        component = regscale_models.Component(
+            title=component_name,
+            componentType=asset.component_type,
+            securityPlansId=self.plan_id,
+            description=component_name,
+            componentOwnerId=self.get_assessor_id(),
+        ).get_or_create()
+        self.components.append(component)
+        return component
+
+    def _handle_component_mapping(self, component: regscale_models.Component) -> None:
+        """
+        Handle component mapping creation if needed.
+
+        :param regscale_models.Component component: The component to create mapping for
+        """
+        if not (component.securityPlansId and not self.is_component):
+            return
+
+        component_mapping = regscale_models.ComponentMapping(
+            componentId=component.id,
+            securityPlanId=self.plan_id,
+        )
+        mapping_result = component_mapping.get_or_create()
+
+        if mapping_result is None:
+            logger.debug(
+                f"Failed to create or find ComponentMapping for componentId={component.id}, securityPlanId={self.plan_id}"
+            )
+        else:
+            mapping_id = getattr(mapping_result, "id", "unknown")
+            logger.debug(f"Successfully handled ComponentMapping for componentId={component.id}, ID={mapping_id}")
+
+    def _handle_component_mapping_and_durosuite(
+        self,
+        existing_or_new_asset: Optional[regscale_models.Asset],
+        component: Optional[regscale_models.Component],
+        asset: IntegrationAsset,
+        created: bool,
+    ) -> None:
+        """
+        Handle component mapping and DuroSuite scanning after asset creation.
+
+        :param Optional[regscale_models.Asset] existing_or_new_asset: The asset that was created/updated
+        :param Optional[regscale_models.Component] component: The associated component, if any
+        :param IntegrationAsset asset: The original integration asset
+        :param bool created: Whether the asset was newly created
+        """
         if existing_or_new_asset and component:
             _was_created, _asset_mapping = regscale_models.AssetMapping(
                 assetId=existing_or_new_asset.id,
@@ -1086,9 +1227,33 @@ class ScannerIntegration(ABC):
             ).get_or_create_with_status()
 
         if created and DuroSuiteVariables.duroSuiteEnabled:
-            # Check if this is a DuroSuite compatible asset
             scan_durosuite_devices(asset=asset, plan_id=self.plan_id, progress=self.asset_progress)
 
+    def _truncate_field(self, value: Optional[str], max_length: int, field_name: str) -> Optional[str]:
+        """
+        Truncate a field to the maximum allowed length to prevent database errors.
+
+        :param Optional[str] value: The value to truncate
+        :param int max_length: Maximum allowed length
+        :param str field_name: Name of the field being truncated (for logging)
+        :return: Truncated value or None
+        :rtype: Optional[str]
+        """
+        if not value:
+            return value
+
+        if len(value) > max_length:
+            truncated = value[:max_length]
+            logger.warning(
+                "Truncated %s field from %d to %d characters for value: %s...",
+                field_name,
+                len(value),
+                max_length,
+                truncated[:100],
+            )
+            return truncated
+        return value
+
     def create_new_asset(
         self, asset: IntegrationAsset, component: Optional[regscale_models.Component]
     ) -> tuple[bool, Optional[regscale_models.Asset]]:
@@ -1101,22 +1266,130 @@ class ScannerIntegration(ABC):
         :return: Tuple of (was_created, newly created asset instance).
         :rtype: tuple[bool, Optional[regscale_models.Asset]]
         """
-        # Ensure the asset has a name
+        if not self._validate_asset_requirements(asset):
+            return False, None
+
+        asset_type = self._validate_and_map_asset_type(asset.asset_type)
+        other_tracking_number = self._prepare_tracking_number(asset)
+        field_data = self._prepare_truncated_asset_fields(asset, other_tracking_number)
+
+        new_asset = self._create_regscale_asset_model(asset, component, asset_type, field_data)
+
+        created, new_asset = new_asset.create_or_update_with_status(bulk_update=True)
+        self.asset_map_by_identifier[asset.identifier] = new_asset
+        logger.debug("Created new asset with identifier %s", asset.identifier)
+
+        self._handle_software_and_stig_processing(new_asset, asset, created)
+        return created, new_asset
+
+    def _validate_asset_requirements(self, asset: IntegrationAsset) -> bool:
+        """Validate that the asset has required fields for creation."""
         if not asset.name:
             logger.warning(
                 "Asset name is required for asset creation. Skipping asset creation of asset_type: %s", asset.asset_type
             )
-            return False, None
+            return False
+        return True
+
+    def _validate_and_map_asset_type(self, asset_type: str) -> str:
+        """Validate and map asset type to valid RegScale values."""
+        valid_asset_types = [
+            "Physical Server",
+            "Virtual Machine (VM)",
+            "Appliance",
+            "Network Router",
+            "Network Switch",
+            "Firewall",
+            "Desktop",
+            "Laptop",
+            "Tablet",
+            "Phone",
+            "Other",
+        ]
+
+        if asset_type not in valid_asset_types:
+            logger.debug(f"Asset type '{asset_type}' not in valid types, mapping to 'Other'")
+            return "Other"
+        return asset_type
+
+    def _prepare_tracking_number(self, asset: IntegrationAsset) -> str:
+        """Prepare and validate the tracking number for asset deduplication."""
+        other_tracking_number = asset.other_tracking_number or asset.identifier
+        if not other_tracking_number:
+            logger.warning("No tracking number available for asset %s, using name as fallback", asset.name)
+            other_tracking_number = asset.name
+        return other_tracking_number
+
+    def _prepare_truncated_asset_fields(self, asset: IntegrationAsset, other_tracking_number: str) -> dict:
+        """Prepare and truncate asset fields to prevent database errors."""
+        max_field_length = 450
+        name = self._process_asset_name(asset, max_field_length)
+
+        return {
+            "name": name,
+            "azure_identifier": self._truncate_field(asset.azure_identifier, max_field_length, "azureIdentifier"),
+            "aws_identifier": self._truncate_field(asset.aws_identifier, max_field_length, "awsIdentifier"),
+            "google_identifier": self._truncate_field(asset.google_identifier, max_field_length, "googleIdentifier"),
+            "other_cloud_identifier": self._truncate_field(
+                asset.other_cloud_identifier, max_field_length, "otherCloudIdentifier"
+            ),
+            "software_name": self._truncate_field(asset.software_name, max_field_length, "softwareName"),
+            "other_tracking_number": self._truncate_field(
+                other_tracking_number, max_field_length, "otherTrackingNumber"
+            ),
+        }
+
+    def _process_asset_name(self, asset: IntegrationAsset, max_field_length: int) -> str:
+        """Process and truncate asset name, handling special cases like Azure resource paths."""
+        name = self._truncate_field(asset.name, max_field_length, "name")
+
+        # For very long Azure resource paths, extract meaningful parts
+        if asset.name and len(asset.name) > max_field_length and "/" in asset.name:
+            name = self._shorten_azure_resource_path(asset.name, max_field_length)
+
+        return name
+
+    def _shorten_azure_resource_path(self, full_name: str, max_field_length: int) -> str:
+        """Shorten long Azure resource paths to meaningful parts."""
+        parts = full_name.split("/")
+        if len(parts) >= 4:
+            # Extract key components from Azure resource path
+            resource_group = next(
+                (p for i, p in enumerate(parts) if i > 0 and parts[i - 1].lower() == "resourcegroups"), ""
+            )
+            resource_type = parts[-2] if len(parts) > 1 else ""
+            resource_name = parts[-1]
+
+            # Build a shortened but meaningful name
+            if resource_group:
+                name = f"../{resource_group}/.../{resource_type}/{resource_name}"
+            else:
+                name = f".../{resource_type}/{resource_name}"
+
+            # Ensure it fits within limits
+            if len(name) > max_field_length:
+                name = name[-(max_field_length):]
+
+            logger.info(
+                "Shortened long Azure resource path from %d to %d characters: %s", len(full_name), len(name), name
+            )
+            return name
+
+        return self._truncate_field(full_name, max_field_length, "name")
 
+    def _create_regscale_asset_model(
+        self, asset: IntegrationAsset, component: Optional[regscale_models.Component], asset_type: str, field_data: dict
+    ) -> regscale_models.Asset:
+        """Create the RegScale Asset model with all required fields."""
         new_asset = regscale_models.Asset(
-            name=asset.name,
+            name=field_data["name"],
             description=asset.description,
             bVirtual=asset.is_virtual,
-            otherTrackingNumber=asset.other_tracking_number or asset.identifier,
-            assetOwnerId=asset.asset_owner_id or "Unknown",
+            otherTrackingNumber=field_data["other_tracking_number"],
+            assetOwnerId=asset.asset_owner_id or regscale_models.Asset.get_user_id() or "Unknown",
             parentId=component.id if component else self.plan_id,
             parentModule=self.parent_module,
-            assetType=asset.asset_type,
+            assetType=asset_type,
             dateLastUpdated=asset.date_last_updated or get_current_datetime(),
             status=asset.status,
             assetCategory=asset.asset_category,
@@ -1127,7 +1400,7 @@ class ScannerIntegration(ABC):
             serialNumber=asset.serial_number,
             assetTagNumber=asset.asset_tag_number,
             bPublicFacing=asset.is_public_facing,
-            azureIdentifier=asset.azure_identifier,
+            azureIdentifier=field_data["azure_identifier"],
             location=asset.location,
             ipAddress=asset.ip_address,
             iPv6Address=asset.ipv6_address,
@@ -1141,13 +1414,13 @@ class ScannerIntegration(ABC):
             endOfLifeDate=asset.end_of_life_date,
             vlanId=asset.vlan_id,
             uri=asset.uri,
-            awsIdentifier=asset.aws_identifier,
-            googleIdentifier=asset.google_identifier,
-            otherCloudIdentifier=asset.other_cloud_identifier,
+            awsIdentifier=field_data["aws_identifier"],
+            googleIdentifier=field_data["google_identifier"],
+            otherCloudIdentifier=field_data["other_cloud_identifier"],
             patchLevel=asset.patch_level,
             cpe=asset.cpe,
             softwareVersion=asset.software_version,
-            softwareName=asset.software_name,
+            softwareName=field_data["software_name"],
             softwareVendor=asset.software_vendor,
             bLatestScan=asset.is_latest_scan,
             bAuthenticatedScan=asset.is_authenticated_scan,
@@ -1156,20 +1429,21 @@ class ScannerIntegration(ABC):
             softwareFunction=asset.software_function,
             baselineConfiguration=asset.baseline_configuration,
         )
+
         if self.asset_identifier_field:
             setattr(new_asset, self.asset_identifier_field, asset.identifier)
 
-        created, new_asset = new_asset.create_or_update_with_status(bulk_update=True)
-        # add to asset_map_by_identifier
-        self.asset_map_by_identifier[asset.identifier] = new_asset
-        logger.debug("Created new asset with identifier %s", asset.identifier)
+        return new_asset
 
+    def _handle_software_and_stig_processing(
+        self, new_asset: regscale_models.Asset, asset: IntegrationAsset, created: bool
+    ) -> None:
+        """Handle post-asset creation tasks like software inventory and STIG mapping."""
         self.handle_software_inventory(new_asset, asset.software_inventory, created)
         self.create_asset_data_and_link(new_asset, asset)
         self.create_or_update_ports_protocol(new_asset, asset)
         if self.stig_mapper:
             self.stig_mapper.map_associated_stigs_to_asset(asset=new_asset, ssp_id=self.plan_id)
-        return created, new_asset
 
     def handle_software_inventory(
         self, new_asset: regscale_models.Asset, software_inventory: List[Dict[str, Any]], created: bool
@@ -1550,35 +1824,96 @@ class ScannerIntegration(ABC):
         finding_id = self.get_finding_identifier(finding)
         finding_id_lock = self._get_lock(finding_id)
 
+        self._log_finding_processing_info(finding, finding_id, issue_status, title)
+
         with finding_id_lock:
-            if ScannerVariables.issueCreation.lower() != "perasset":
-                # Check if we should consolidate open issues based on integrationFindingId
-                if issue_status == regscale_models.IssueStatus.Open:
-                    existing_issues = regscale_models.Issue.find_by_integration_finding_id(finding_id)
-                    # Find an open issue to update
-                    issue = next(
-                        (issue for issue in existing_issues if issue.status != regscale_models.IssueStatus.Closed), None
-                    )
-                    if issue:
-                        return self._create_or_update_issue(finding, issue_status, title, issue)
-
-                # Check if we should consolidate closed issues based on integrationFindingId and issueDueDates
-                elif issue_status == regscale_models.IssueStatus.Closed:
-                    existing_issues = regscale_models.Issue.find_by_integration_finding_id(finding_id)
-                    # Find a closed issue with matching due date to consolidate with
-                    matching_closed_issue = next(
-                        (
-                            issue
-                            for issue in existing_issues
-                            if issue.status == regscale_models.IssueStatus.Closed
-                            and date_str(issue.dueDate) == date_str(finding.due_date)
-                        ),
-                        None,
-                    )
-                    if matching_closed_issue:
-                        return self._create_or_update_issue(finding, issue_status, title, matching_closed_issue)
+            existing_issue = self._find_existing_issue_for_finding(finding_id, finding, issue_status)
+            return self._create_or_update_issue(finding, issue_status, title, existing_issue)
+
+    def _log_finding_processing_info(
+        self, finding: IntegrationFinding, finding_id: str, issue_status: regscale_models.IssueStatus, title: str
+    ) -> None:
+        """Log finding processing information for debugging."""
+        logger.debug(
+            f"PROCESSING FINDING: external_id={finding.external_id}, finding_id={finding_id}, status={issue_status}, title='{title[:50]}...'"
+        )
 
-            return self._create_or_update_issue(finding, issue_status, title)
+        if issue_status == regscale_models.IssueStatus.Closed:
+            logger.debug(f"CLOSED FINDING: This will create/update a CLOSED issue (status={issue_status})")
+
+    def _find_existing_issue_for_finding(
+        self, finding_id: str, finding: IntegrationFinding, issue_status: regscale_models.IssueStatus
+    ) -> Optional[regscale_models.Issue]:
+        """Find existing issue for the finding based on status and creation type."""
+        if ScannerVariables.issueCreation.lower() == "perasset":
+            return None
+
+        existing_issues = self._get_existing_issues_for_finding(finding_id, finding)
+
+        if issue_status == regscale_models.IssueStatus.Open:
+            return self._find_issue_for_open_status(existing_issues, finding_id)
+        elif issue_status == regscale_models.IssueStatus.Closed:
+            return self._find_issue_for_closed_status(existing_issues, finding, finding_id)
+
+        return None
+
+    def _get_existing_issues_for_finding(
+        self, finding_id: str, finding: IntegrationFinding
+    ) -> List[regscale_models.Issue]:
+        """Get existing issues for the finding using various lookup methods."""
+        existing_issues = regscale_models.Issue.find_by_integration_finding_id(finding_id)
+
+        # If no issues found by integrationFindingId, try fallback lookup by identifier fields
+        if not existing_issues and finding.external_id:
+            existing_issues = self._find_issues_by_identifier_fallback(finding.external_id)
+
+        return existing_issues
+
+    def _find_issue_for_open_status(
+        self, existing_issues: List[regscale_models.Issue], finding_id: str
+    ) -> Optional[regscale_models.Issue]:
+        """Find appropriate issue when the finding status is Open."""
+        # Find an open issue to update first
+        open_issue = next(
+            (issue for issue in existing_issues if issue.status != regscale_models.IssueStatus.Closed), None
+        )
+        if open_issue:
+            return open_issue
+
+        # If no open issue found, look for a closed issue to reopen
+        closed_issue = next(
+            (issue for issue in existing_issues if issue.status == regscale_models.IssueStatus.Closed), None
+        )
+        if closed_issue:
+            logger.debug(f"Reopening closed issue {closed_issue.id} for finding {finding_id}")
+            return closed_issue
+
+        return None
+
+    def _find_issue_for_closed_status(
+        self, existing_issues: List[regscale_models.Issue], finding: IntegrationFinding, finding_id: str
+    ) -> Optional[regscale_models.Issue]:
+        """Find appropriate issue when the finding status is Closed."""
+        # Find a closed issue with matching due date to consolidate with
+        matching_closed_issue = next(
+            (
+                issue
+                for issue in existing_issues
+                if issue.status == regscale_models.IssueStatus.Closed
+                and date_str(issue.dueDate) == date_str(finding.due_date)
+            ),
+            None,
+        )
+        if matching_closed_issue:
+            return matching_closed_issue
+
+        # If no matching closed issue, look for any existing issue to update
+        any_existing_issue = next(iter(existing_issues), None) if existing_issues else None
+        if any_existing_issue:
+            logger.debug(f"Closing existing issue {any_existing_issue.id} for finding {finding_id}")
+            return any_existing_issue
+
+        return None
 
     def _create_or_update_issue(
         self,
@@ -1639,6 +1974,90 @@ class ScannerIntegration(ABC):
         self._handle_property_and_milestone_creation(issue, finding, existing_issue)
         return issue
 
+    def _find_issues_by_identifier_fallback(self, external_id: str) -> List[regscale_models.Issue]:
+        """
+        Find issues by identifier fields (otherIdentifier or integration-specific field) as fallback.
+        This helps with deduplication when integrationFindingId lookup fails.
+
+        :param str external_id: The external ID to search for
+        :return: List of matching issues
+        :rtype: List[regscale_models.Issue]
+        """
+        fallback_issues = []
+
+        try:
+            # Get all issues for this plan/component
+            all_issues = regscale_models.Issue.get_all_by_parent(
+                parent_id=self.plan_id,
+                parent_module=self.parent_module,
+            )
+
+            # Filter by source report to only check our integration's issues
+            source_issues = [issue for issue in all_issues if issue.sourceReport == self.title]
+
+            # Look for matches by otherIdentifier
+            for issue in source_issues:
+                if getattr(issue, "otherIdentifier", None) == external_id:
+                    fallback_issues.append(issue)
+                    logger.debug(f"Found issue {issue.id} by otherIdentifier fallback: {external_id}")
+
+                # Also check integration-specific identifier field if configured
+                elif (
+                    self.issue_identifier_field
+                    and hasattr(issue, self.issue_identifier_field)
+                    and getattr(issue, self.issue_identifier_field) == external_id
+                ):
+                    fallback_issues.append(issue)
+                    logger.debug(f"Found issue {issue.id} by {self.issue_identifier_field} fallback: {external_id}")
+
+            if fallback_issues:
+                logger.info(
+                    f"Fallback deduplication found {len(fallback_issues)} existing issue(s) for external_id: {external_id}"
+                )
+
+        except Exception as e:
+            logger.warning(f"Error in fallback issue lookup for {external_id}: {e}")
+
+        return fallback_issues
+
+    def _set_issue_identifier_fields_internal(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
+        """Set issue identifier fields (e.g., wizId) on the issue object without saving."""
+        if not finding.external_id:
+            logger.debug(f"finding.external_id is empty: {finding.external_id}")
+            return
+
+        logger.debug(f"Setting issue identifier fields: external_id={finding.external_id}")
+
+        # Set otherIdentifier field (the external ID field in Issue model)
+        if not getattr(issue, "otherIdentifier", None):  # Only set if not already set
+            issue.otherIdentifier = finding.external_id
+            logger.debug(f"Set otherIdentifier = {finding.external_id}")
+
+        # Set the specific identifier field if configured (e.g., wizId for Wiz)
+        if self.issue_identifier_field and hasattr(issue, self.issue_identifier_field):
+            current_value = getattr(issue, self.issue_identifier_field)
+            if not current_value:  # Only set if not already set
+                setattr(issue, self.issue_identifier_field, finding.external_id)
+                logger.debug(f"Set {self.issue_identifier_field} = {finding.external_id}")
+            else:
+                logger.debug(f"{self.issue_identifier_field} already set to: {current_value}")
+        else:
+            if self.issue_identifier_field:  # Only log warning if field is configured
+                logger.warning(
+                    f"Cannot set issue_identifier_field: field='{self.issue_identifier_field}', hasattr={hasattr(issue, self.issue_identifier_field)}"
+                )
+
+    def _set_issue_identifier_fields(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
+        """Set issue identifier fields (e.g., wizId) and save them to the database."""
+        self._set_issue_identifier_fields_internal(issue, finding)
+
+        # Explicitly save the issue to persist the identifier fields
+        try:
+            issue.save(bulk=True)
+            logger.info(f"Saved issue {issue.id} with identifier fields")
+        except Exception as e:
+            logger.error(f"Failed to save issue identifier fields: {e}")
+
     def _set_basic_issue_fields(
         self,
         issue: regscale_models.Issue,
@@ -1664,25 +2083,31 @@ class ScannerIntegration(ABC):
         issue.securityPlanId = self.plan_id if not self.is_component else None
         issue.identification = finding.identification
         issue.dateFirstDetected = finding.first_seen
-        issue.assetIdentifier = asset_identifier
+        issue.assetIdentifier = finding.issue_asset_identifier_value or asset_identifier
+
+        # Set organization ID based on Issue Owner or SSP Owner hierarchy
+        issue.orgId = self.determine_issue_organization_id(issue.issueOwnerId)
 
     def _set_issue_due_date(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
-        """Set the due date for the issue."""
+        """Set the due date for the issue using DueDateHandler."""
         if not finding.due_date:
             try:
                 base_created = finding.date_created or issue.dateCreated
-                finding.due_date = issue_due_date(
+                finding.due_date = self.due_date_handler.calculate_due_date(
                     severity=finding.severity,
                     created_date=base_created,
-                    title=self.title,
+                    cve=finding.cve,
+                    title=finding.title or self.title,
                 )
-            except Exception:
+            except Exception as e:
+                logger.warning(f"Error calculating due date with DueDateHandler: {e}")
                 # Final fallback to a Low severity default if anything goes wrong
                 base_created = finding.date_created or issue.dateCreated
-                finding.due_date = issue_due_date(
+                finding.due_date = self.due_date_handler.calculate_due_date(
                     severity=regscale_models.IssueSeverity.Low,
                     created_date=base_created,
-                    title=self.title,
+                    cve=finding.cve,
+                    title=finding.title or self.title,
                 )
         issue.dueDate = finding.due_date
 
@@ -1700,6 +2125,9 @@ class ScannerIntegration(ABC):
         issue.cve = finding.cve
         issue.assessmentId = finding.assessment_id
 
+        # Set issue identifier fields (e.g., wizId, otherIdentifier) before save/create
+        self._set_issue_identifier_fields_internal(issue, finding)
+
     def _set_control_fields(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
         """Set control-related fields for the issue."""
         control_id = self.get_control_implementation_id_for_cci(finding.cci_ref) if finding.cci_ref else None
@@ -1743,13 +2171,21 @@ class ScannerIntegration(ABC):
     ) -> None:
         """Save or create the issue."""
         if existing_issue:
+            logger.debug(f"UPDATING EXISTING ISSUE: {existing_issue.id} with external_id={finding.external_id}")
             logger.debug("Saving Old Issue: %s  with assetIdentifier: %s", issue.id, issue.assetIdentifier)
             issue.save(bulk=True)
             logger.debug("Saved existing issue %s with assetIdentifier: %s", issue.id, issue.assetIdentifier)
         else:
+            logger.debug(
+                f"➕ CREATING NEW ISSUE: external_id={finding.external_id}, title='{finding.title[:50]}...', status={finding.status}"
+            )
             issue = issue.create_or_update(
                 bulk_update=True, defaults={"otherIdentifier": self._get_other_identifier(finding, is_poam)}
             )
+            if issue.id:
+                logger.debug(f"NEW ISSUE CREATED: RegScale ID={issue.id}, external_id={finding.external_id}")
+            else:
+                logger.warning(f"ISSUE CREATION FAILED: No ID assigned for external_id={finding.external_id}")
             self.extra_data_to_properties(finding, issue.id)
 
         self._handle_property_and_milestone_creation(issue, finding, existing_issue)
@@ -1766,65 +2202,137 @@ class ScannerIntegration(ABC):
 
         :param regscale_models.Issue issue: The issue to handle properties for
         :param IntegrationFinding finding: The finding data
-        :param bool new_issue: Whether this is a new issue
+        :param Optional[regscale_models.Issue] existing_issue: Existing issue for milestone comparison
         :rtype: None
         """
+        # Handle property creation
+        self._create_issue_properties(issue, finding)
+
+        # Handle milestone creation
+        self._create_issue_milestones(issue, finding, existing_issue)
+
+    def _create_issue_properties(self, issue: regscale_models.Issue, finding: IntegrationFinding) -> None:
+        """
+        Create properties for an issue based on finding data.
+
+        :param regscale_models.Issue issue: The issue to create properties for
+        :param IntegrationFinding finding: The finding data
+        """
         if poc := finding.point_of_contact:
-            regscale_models.Property(
-                key="POC",
-                value=poc,
-                parentId=issue.id,
-                parentModule="issues",
-            ).create_or_update()
-            logger.debug("Added POC property %s to issue %s", poc, issue.id)
+            self._create_property_safe(issue, "POC", poc, "POC property")
 
         if finding.is_cwe:
+            self._create_property_safe(issue, "CWE", finding.plugin_id, "CWE property")
+
+    def _create_property_safe(self, issue: regscale_models.Issue, key: str, value: str, property_type: str) -> None:
+        """
+        Safely create a property with error handling.
+
+        :param regscale_models.Issue issue: The issue to create property for
+        :param str key: The property key
+        :param str value: The property value
+        :param str property_type: Description for logging purposes
+        """
+        try:
             regscale_models.Property(
-                key="CWE",
-                value=finding.plugin_id,
+                key=key,
+                value=value,
                 parentId=issue.id,
                 parentModule="issues",
             ).create_or_update()
-            logger.debug("Added CWE property %s to issue %s", finding.plugin_id, issue.id)
+            logger.debug("Added %s %s to issue %s", property_type, value, issue.id)
+        except Exception as e:
+            logger.warning("Failed to create %s: %s", property_type, str(e))
 
-        if ScannerVariables.useMilestones and issue.id:
-            if (
-                existing_issue
-                and existing_issue.status == regscale_models.IssueStatus.Closed
-                and issue.status == regscale_models.IssueStatus.Open
-            ):
-                regscale_models.Milestone(
-                    title=f"Issue reopened from {self.title} scan",
-                    milestoneDate=get_current_datetime(),
-                    responsiblePersonId=self.assessor_id,
-                    parentID=issue.id,
-                    parentModule="issues",
-                ).create()
-                logger.debug("Added milestone for issue %s from finding %s", issue.id, finding.external_id)
-            elif (
-                existing_issue
-                and existing_issue.status == regscale_models.IssueStatus.Open
-                and issue.status == regscale_models.IssueStatus.Closed
-            ):
-                regscale_models.Milestone(
-                    title=f"Issue closed from {self.title} scan",
-                    milestoneDate=issue.dateCompleted,
-                    responsiblePersonId=self.assessor_id,
-                    parentID=issue.id,
-                    parentModule="issues",
-                ).create()
-                logger.debug("Added milestone for issue %s from finding %s", issue.id, finding.external_id)
-            elif not existing_issue:
-                regscale_models.Milestone(
-                    title=f"Issue created from {self.title} scan",
-                    milestoneDate=self.scan_date,
-                    responsiblePersonId=self.assessor_id,
-                    parentID=issue.id,
-                    parentModule="issues",
-                ).create()
-                logger.debug("Created milestone for issue %s from finding %s", issue.id, finding.external_id)
-            else:
-                logger.debug("No milestone created for issue %s from finding %s", issue.id, finding.external_id)
+    def _create_issue_milestones(
+        self,
+        issue: regscale_models.Issue,
+        finding: IntegrationFinding,
+        existing_issue: Optional[regscale_models.Issue],
+    ) -> None:
+        """
+        Create milestones for an issue based on status transitions.
+
+        :param regscale_models.Issue issue: The issue to create milestones for
+        :param IntegrationFinding finding: The finding data
+        :param Optional[regscale_models.Issue] existing_issue: Existing issue for comparison
+        """
+        if not (ScannerVariables.useMilestones and issue.id):
+            return
+
+        if self._should_create_reopened_milestone(existing_issue, issue):
+            self._create_milestone_safe(
+                issue, finding, "Issue reopened from", get_current_datetime(), "reopened milestone"
+            )
+        elif self._should_create_closed_milestone(existing_issue, issue):
+            self._create_milestone_safe(issue, finding, "Issue closed from", issue.dateCompleted, "closed milestone")
+        elif not existing_issue:
+            self._create_milestone_safe(issue, finding, "Issue created from", self.scan_date, "new issue milestone")
+        else:
+            logger.debug("No milestone created for issue %s from finding %s", issue.id, finding.external_id)
+
+    def _should_create_reopened_milestone(
+        self, existing_issue: Optional[regscale_models.Issue], issue: regscale_models.Issue
+    ) -> bool:
+        """
+        Check if a reopened milestone should be created.
+
+        :param Optional[regscale_models.Issue] existing_issue: The existing issue
+        :param regscale_models.Issue issue: The current issue
+        :return: True if reopened milestone should be created
+        :rtype: bool
+        """
+        return (
+            existing_issue
+            and existing_issue.status == regscale_models.IssueStatus.Closed
+            and issue.status == regscale_models.IssueStatus.Open
+        )
+
+    def _should_create_closed_milestone(
+        self, existing_issue: Optional[regscale_models.Issue], issue: regscale_models.Issue
+    ) -> bool:
+        """
+        Check if a closed milestone should be created.
+
+        :param Optional[regscale_models.Issue] existing_issue: The existing issue
+        :param regscale_models.Issue issue: The current issue
+        :return: True if closed milestone should be created
+        :rtype: bool
+        """
+        return (
+            existing_issue
+            and existing_issue.status == regscale_models.IssueStatus.Open
+            and issue.status == regscale_models.IssueStatus.Closed
+        )
+
+    def _create_milestone_safe(
+        self,
+        issue: regscale_models.Issue,
+        finding: IntegrationFinding,
+        title_prefix: str,
+        milestone_date: str,
+        milestone_type: str,
+    ) -> None:
+        """
+        Safely create a milestone with error handling.
+
+        :param regscale_models.Issue issue: The issue to create milestone for
+        :param IntegrationFinding finding: The finding data
+        :param str title_prefix: Prefix for milestone title
+        :param str milestone_date: Date for the milestone
+        :param str milestone_type: Description for logging purposes
+        """
+        try:
+            regscale_models.Milestone(
+                title=f"{title_prefix} {self.title} scan",
+                milestoneDate=milestone_date,
+                responsiblePersonId=self.assessor_id,
+                parentID=issue.id,
+                parentModule="issues",
+            ).create_or_update()
+            logger.debug("Added milestone for issue %s from finding %s", issue.id, finding.external_id)
+        except Exception as e:
+            logger.warning("Failed to create %s: %s", milestone_type, str(e))
 
     @staticmethod
     def extra_data_to_properties(finding: IntegrationFinding, issue_id: int) -> None:
@@ -1870,13 +2378,17 @@ class ScannerIntegration(ABC):
         :rtype: str
         """
         delimiter = "\n"
+
+        # Use issue_asset_identifier_value if available (e.g., providerUniqueId from Wiz)
+        # This provides more meaningful asset identification for eMASS exports
+        current_asset_identifier = finding.issue_asset_identifier_value or finding.asset_identifier
         if not existing_issue or ScannerVariables.issueCreation.lower() == "perasset":
-            return finding.asset_identifier
+            return current_asset_identifier
 
         # Get existing asset identifiers
         existing_asset_identifiers = set((existing_issue.assetIdentifier or "").split(delimiter))
-        if finding.asset_identifier not in existing_asset_identifiers:
-            existing_asset_identifiers.add(finding.asset_identifier)
+        if current_asset_identifier not in existing_asset_identifiers:
+            existing_asset_identifiers.add(current_asset_identifier)
 
         return delimiter.join(existing_asset_identifiers)
 
@@ -1907,16 +2419,14 @@ class ScannerIntegration(ABC):
         """
         Determine if the cve is part of the published CISA KEV list
 
+        Note: Due date handling is now managed by DueDateHandler. This method only sets kevList field.
+
         :param str cve: The CVE to lookup in CISAs KEV list
-        :param regscale_models.Issue issue: The issue to update kevList field and dueDate if found in KEV List
+        :param regscale_models.Issue issue: The issue to update kevList field
         :param Optional[ThreadSafeDict[str, Any]] cisa_kevs: The CISA KEV data to search the findings
         :return: The updated issue
         :rtype: regscale_models.Issue
         """
-        from datetime import datetime
-
-        from regscale.core.app.utils.app_utils import convert_datetime_to_regscale_string
-
         issue.kevList = "No"
 
         if cisa_kevs:
@@ -1929,14 +2439,6 @@ class ScannerIntegration(ABC):
                 None,
             )
             if kev_data:
-                # If kev due date is before the issue date created, add the difference to the date created
-                calculated_due_date = ScannerIntegration._calculate_kev_due_date(kev_data, issue.dateCreated)
-                if calculated_due_date:
-                    issue.dueDate = calculated_due_date
-                else:
-                    issue.dueDate = convert_datetime_to_regscale_string(
-                        datetime.strptime(kev_data["dueDate"], "%Y-%m-%d")
-                    )
                 issue.kevList = "Yes"
 
         return issue
@@ -2245,6 +2747,9 @@ class ScannerIntegration(ABC):
         # Finalize processing
         self._finalize_finding_processing(scan_history, current_vulnerabilities)
 
+        # Complete the finding progress bar
+        self._complete_finding_progress(loading_findings, processed_findings_count)
+
         return processed_findings_count
 
     def _setup_finding_progress(self):
@@ -2298,6 +2803,15 @@ class ScannerIntegration(ABC):
             )
         self.finding_progress.advance(loading_findings, 1)
 
+    def _complete_finding_progress(self, loading_findings, processed_count):
+        """Complete the finding progress bar with final status."""
+        self.finding_progress.update(
+            loading_findings,
+            completed=processed_count,
+            total=max(processed_count, self.num_findings_to_process or processed_count),
+            description=f"[green] Completed processing {processed_count} finding(s) from {self.title}",
+        )
+
     def _process_findings_in_batches(
         self, findings: Iterator[IntegrationFinding], process_finding_with_progress
     ) -> int:
@@ -2584,7 +3098,7 @@ class ScannerIntegration(ABC):
                 logger.debug(f"Creating vulnerability for finding {finding.external_id} (attempt {attempt + 1})")
                 vulnerability = self.create_vulnerability_from_finding(finding, asset, scan_history)
                 finding.vulnerability_id = vulnerability.id
-                logger.info(f"Successfully created vulnerability {vulnerability.id} for finding {finding.external_id}")
+                logger.debug(f"Successfully created vulnerability {vulnerability.id} for finding {finding.external_id}")
 
                 if ScannerVariables.vulnerabilityCreation.lower() != "noissue":
                     # Handle associated issue
@@ -2797,6 +3311,14 @@ class ScannerIntegration(ABC):
             logger.info("Skipping closing outdated issues.")
             return 0
 
+        # Check global preventAutoClose setting
+        from regscale.core.app.application import Application
+
+        app = Application()
+        if app.config.get("preventAutoClose", False):
+            logger.info("Skipping closing outdated issues due to global preventAutoClose setting.")
+            return 0
+
         closed_count = 0
         affected_control_ids = set()
         count_lock = threading.Lock()
@@ -2854,14 +3376,17 @@ class ScannerIntegration(ABC):
         issue.save()
 
         if ScannerVariables.useMilestones and issue.id:
-            regscale_models.Milestone(
-                title=f"Issue closed from {self.title} scan",
-                milestoneDate=issue.dateCompleted,
-                responsiblePersonId=self.assessor_id,
-                completed=True,
-                parentID=issue.id,
-                parentModule="issues",
-            ).create()
+            try:
+                regscale_models.Milestone(
+                    title=f"Issue closed from {self.title} scan",
+                    milestoneDate=issue.dateCompleted,
+                    responsiblePersonId=self.assessor_id,
+                    completed=True,
+                    parentID=issue.id,
+                    parentModule="issues",
+                ).create_or_update()
+            except Exception as e:
+                logger.warning("Failed to create closed issue milestone: %s", str(e))
             logger.debug("Created milestone for issue %s from %s tool", issue.id, self.title)
 
         with count_lock:
@@ -2929,6 +3454,41 @@ class ScannerIntegration(ABC):
             self.control_implementation_map[control_id] = control_implementation.save()
             logger.info("Updated control implementation %d status to %s", control_id, new_status)
 
+    def is_issue_protected_from_auto_close(self, issue: regscale_models.Issue) -> bool:
+        """
+        Check if an issue is protected from automatic closure.
+
+        :param regscale_models.Issue issue: The issue to check
+        :return: True if the issue should not be auto-closed
+        :rtype: bool
+        """
+        try:
+            # Check global configuration setting
+            app = Application()
+            if app.config.get("preventAutoClose", False):
+                logger.debug(f"Issue {issue.id} is protected from auto-closure by global preventAutoClose setting")
+                return True
+
+            # Check for protection property
+            properties = Property.get_all_by_parent(parent_id=issue.id, parent_module="issues")
+
+            for prop in properties:
+                if prop.key == "PREVENT_AUTO_CLOSE" and prop.value.lower() == "true":
+                    logger.debug(f"Issue {issue.id} is protected from auto-closure by PREVENT_AUTO_CLOSE property")
+                    return True
+
+            # Check for manual reopen indicators in changes
+            if issue.changes and "manually reopened" in issue.changes.lower():
+                logger.debug(f"Issue {issue.id} is protected from auto-closure due to manual reopen indicator")
+                return True
+
+            return False
+
+        except Exception as e:
+            # If we can't check, err on the side of caution and protect the issue
+            logger.warning(f"Could not check protection status for issue {issue.id}: {e}")
+            return True
+
     def should_close_issue(self, issue: regscale_models.Issue, current_vulnerabilities: Dict[int, Set[int]]) -> bool:
         """
         Determines if an issue should be closed based on current vulnerabilities.
@@ -2946,6 +3506,11 @@ class ScannerIntegration(ABC):
             )
             return False
 
+        # Check if the issue is protected from auto-closure
+        if self.is_issue_protected_from_auto_close(issue):
+            logger.debug(f"Issue {issue.id} is protected from automatic closure")
+            return False
+
         # If the issue has a vulnerability ID, check if it's still current for any asset
         if issue.vulnerabilityId:
             # Get vulnerability mappings for this issue
@@ -3316,11 +3881,11 @@ class ScannerIntegration(ABC):
         :return: None
         :rtype: None
         """
-        finding.due_date = issue_due_date(
+        finding.due_date = self.due_date_handler.calculate_due_date(
             severity=finding.severity,
             created_date=finding.date_created or self.scan_date,
-            title=self.title,
-            config=self.app.config,
+            cve=finding.cve,
+            title=finding.title or self.title,
         )
 
     def _update_last_seen_date(self, finding: IntegrationFinding) -> None: