regscale-cli 6.21.1.0__py3-none-any.whl → 6.21.2.1__py3-none-any.whl

regscale/integrations/compliance_integration.py

@@ -114,6 +114,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         create_issues: bool = True,
         update_control_status: bool = True,
         create_poams: bool = False,
+        parent_module: str = "securityplans",
         **kwargs,
     ):
         """
@@ -154,7 +155,8 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         self._impl_id_by_control: Dict[str, int] = {}
         # Key: ControlImplementation.id -> Assessment created/updated today
         self._assessment_by_impl_today: Dict[int, regscale_models.Assessment] = {}
-
+        # suppress asset not found errors in non-debug modes
+        self.suppress_asset_not_found_errors = logger.level != logging.DEBUG
         # Set scan date
         self.scan_date = get_current_datetime()
 
@@ -199,7 +201,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             self._load_existing_assessments()
 
             self._cache_loaded = True
-            logger.info("🗄️  Loaded existing records cache to prevent duplicates:")
+            logger.info("Loaded existing records cache to prevent duplicates:")
             logger.info(f"   - Assets: {len(self._existing_assets_cache)}")
             logger.info(f"   - Issues: {len(self._existing_issues_cache)}")
             logger.info(f"   - Assessments: {len(self._existing_assessments_cache)}")
@@ -238,22 +240,59 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         """
         Load existing issues into cache.
 
+        Uses both plan-level and control-level queries to ensure all relevant issues are found.
+
         :return: None
         :rtype: None
         """
         try:
-            # Get all issues for this plan
-            existing_issues = regscale_models.Issue.get_all_by_parent(
+            all_issues = set()
+
+            # Method 1: Get issues directly associated with the plan
+            plan_issues = regscale_models.Issue.get_all_by_parent(
                 parent_id=self.plan_id, parent_module=self.parent_module
             )
+            all_issues.update(plan_issues)
+            logger.debug(f"Found {len(plan_issues)} issues directly under plan {self.plan_id}")
 
-            for issue in existing_issues:
+            # Method 2: Get issues associated with control implementations (matches scanner integration logic)
+            try:
+                issues_by_impl = regscale_models.Issue.get_open_issues_ids_by_implementation_id(
+                    plan_id=self.plan_id, is_component=getattr(self, "is_component", False)
+                )
+                impl_issues_count = 0
+                for impl_id, issue_list in issues_by_impl.items():
+                    for issue_dict in issue_list:
+                        # issue_dict contains issue data, need to get the actual issue object
+                        issue_id = issue_dict.get("id")
+                        if issue_id:
+                            try:
+                                issue = regscale_models.Issue.get_object(object_id=issue_id)
+                                if issue:
+                                    all_issues.add(issue)
+                                    impl_issues_count += 1
+                            except Exception as e:
+                                logger.debug(f"Could not load issue {issue_id}: {e}")
+
+                logger.debug(f"Found {impl_issues_count} additional issues via control implementations")
+            except Exception as e:
+                logger.debug(f"Could not load issues by control implementation: {e}")
+
+            logger.debug(f"Total unique issues found: {len(all_issues)} for plan {self.plan_id}")
+
+            wiz_issues = 0
+            for issue in all_issues:
                 # Cache by external_id and other_identifier for flexible lookup
                 if hasattr(issue, "externalId") and issue.externalId:
                     self._existing_issues_cache[issue.externalId] = issue
+                    if "wiz-policy" in issue.externalId.lower():
+                        wiz_issues += 1
+                        logger.debug(f"Cached Wiz issue: {issue.id} -> external_id: {issue.externalId}")
                 if hasattr(issue, "otherIdentifier") and issue.otherIdentifier:
                     self._existing_issues_cache[issue.otherIdentifier] = issue
 
+            logger.debug(f"Cached {wiz_issues} Wiz policy issues out of {len(all_issues)} total issues")
+
         except Exception as e:
             logger.debug(f"Error loading existing issues: {e}")
 
@@ -435,7 +474,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 logger.error(f"Error processing compliance item: {e}")
                 continue
 
-        logger.info(
+        logger.debug(
             f"Processed {len(self.all_compliance_items)} compliance items: "
             f"{len(self.all_compliance_items) - len(self.failed_compliance_items)} passing, "
             f"{len(self.failed_compliance_items)} failing"
@@ -592,61 +631,13 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         logger.info(f"Starting {self.title} compliance sync...")
 
         try:
-            # Create scan history
             scan_history = self.create_scan_history()
-
-            # Process compliance data
             self.process_compliance_data()
 
-            # Create assets for all compliance items
-            assets = list(self.fetch_assets())
-            if assets:
-                assets_processed = self.update_regscale_assets(iter(assets))
-                # Use batch results recorded during bulk_save
-                results = getattr(self, "_results", {}).get("assets", {})
-                created = results.get("created_count", 0)
-                updated = results.get("updated_count", 0)
-                deleted = results.get("deleted_count", 0) if isinstance(results, dict) else 0
-                if deleted:
-                    logger.info(
-                        f"Assets processed: {assets_processed} (created: {created}, updated: {updated}, deleted: {deleted})"
-                    )
-                else:
-                    logger.info(f"Assets processed: {assets_processed} (created: {created}, updated: {updated})")
-
-            # Create/update control assessments first so issues can link controlId/assessmentId
-            if self.update_control_status:
-                self._process_control_assessments()
-
-            # Create issues only (no vulnerabilities) for failed compliance items
-            if self.create_issues:
-                findings = list(self.fetch_findings())
-                if findings:
-                    for finding in findings:
-                        try:
-                            asset = self.get_asset_by_identifier(finding.asset_identifier)
-                            if not asset:
-                                # Attempt to create the asset on-demand from cached compliance data
-                                asset = self._ensure_asset_for_finding(finding)
-                            if not asset:
-                                logger.error(
-                                    f"Asset not found for identifier {finding.asset_identifier} — "
-                                    "skipping issue creation for this finding"
-                                )
-                                continue
-
-                            # Directly create/update issue without vulnerability processing
-                            issue_title = self.get_issue_title(finding)
-                            self.create_or_update_issue_from_finding(title=issue_title, finding=finding)
-                        except Exception as e:
-                            logger.error(f"Error processing finding: {e}")
-
-            # Do not write scan history for Wiz policy compliance when disabled
-            try:
-                if getattr(self, "enable_scan_history", True):
-                    self._update_scan_history(scan_history)
-            except Exception:
-                self._update_scan_history(scan_history)
+            self._sync_assets()
+            self._sync_control_assessments()
+            self._sync_issues()
+            self._finalize_scan_history(scan_history)
 
             logger.info(f"Completed {self.title} compliance sync")
 
@@ -654,6 +645,166 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.error(f"Error during compliance sync: {e}")
             raise
 
+    def _sync_assets(self) -> None:
+        """
+        Process and sync assets from compliance items.
+
+        :return: None
+        :rtype: None
+        """
+        assets = list(self.fetch_assets())
+        if not assets:
+            logger.debug("No assets generated from compliance items")
+            return
+
+        assets_processed = self.update_regscale_assets(iter(assets))
+        self._log_asset_results(assets_processed)
+
+    def _log_asset_results(self, assets_processed: int) -> None:
+        """
+        Log asset processing results.
+
+        :param int assets_processed: Number of assets processed
+        :return: None
+        :rtype: None
+        """
+        results = getattr(self, "_results", {}).get("assets", {})
+        created = results.get("created_count", 0)
+        updated = results.get("updated_count", 0)
+        deleted = results.get("deleted_count", 0) if isinstance(results, dict) else 0
+
+        if deleted > 0:
+            logger.info(
+                f"Assets processed: {assets_processed} (created: {created}, updated: {updated}, deleted: {deleted})"
+            )
+        elif created > 0 or updated > 0:
+            logger.info(f"Assets processed: {assets_processed} (created: {created}, updated: {updated})")
+        else:
+            logger.debug(f"Assets processed: {assets_processed} (no changes made)")
+
+    def _sync_control_assessments(self) -> None:
+        """
+        Process control assessments if enabled.
+
+        :return: None
+        :rtype: None
+        """
+        if self.update_control_status:
+            self._process_control_assessments()
+
+    def _sync_issues(self) -> None:
+        """
+        Process and sync issues from failed compliance items.
+
+        :return: None
+        :rtype: None
+        """
+        if not self.create_issues:
+            return
+
+        findings = list(self.fetch_findings())
+        if not findings:
+            logger.debug("No findings to process into issues")
+            return
+
+        issues_created, issues_skipped = self._process_findings_to_issues(findings)
+        self._log_issue_results(issues_created, issues_skipped)
+
+    def _process_findings_to_issues(self, findings: List[IntegrationFinding]) -> tuple[int, int]:
+        """
+        Process findings into issues and return counts.
+
+        :param findings: List of findings to process
+        :return: Tuple of (issues_created, issues_skipped)
+        """
+        issues_created = 0
+        issues_skipped = 0
+
+        for finding in findings:
+            try:
+                if self._process_single_finding(finding):
+                    issues_created += 1
+                else:
+                    issues_skipped += 1
+            except Exception as e:
+                logger.error(f"Error processing finding: {e}")
+                issues_skipped += 1
+
+        return issues_created, issues_skipped
+
+    def _process_single_finding(self, finding: IntegrationFinding) -> bool:
+        """
+        Process a single finding into an issue.
+
+        :param finding: Finding to process
+        :return: True if issue was created/updated, False if skipped
+        """
+        asset = self._get_or_create_asset_for_finding(finding)
+        if not asset:
+            self._log_asset_not_found_error(finding)
+            return False
+
+        issue_title = self.get_issue_title(finding)
+        issue = self.create_or_update_issue_from_finding(title=issue_title, finding=finding)
+        return issue is not None
+
+    def _get_or_create_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
+        """
+        Get existing asset or create one on-demand for the finding.
+
+        :param IntegrationFinding finding: Finding needing an asset
+        :return: Asset if found/created, None otherwise
+        :rtype: Optional[regscale_models.Asset]
+        """
+        asset = self.get_asset_by_identifier(finding.asset_identifier)
+        if not asset:
+            asset = self._ensure_asset_for_finding(finding)
+        return asset
+
+    def _log_asset_not_found_error(self, finding: IntegrationFinding) -> None:
+        """
+        Log error when asset is not found for a finding.
+
+        :param IntegrationFinding finding: Finding with missing asset
+        :return: None
+        :rtype: None
+        """
+        if not getattr(self, "suppress_asset_not_found_errors", False):
+            logger.error(
+                f"Asset not found for identifier {finding.asset_identifier} — "
+                "skipping issue creation for this finding"
+            )
+
+    def _log_issue_results(self, issues_created: int, issues_skipped: int) -> None:
+        """
+        Log issue processing results.
+
+        :param int issues_created: Number of issues created/updated
+        :param int issues_skipped: Number of issues skipped
+        :return: None
+        :rtype: None
+        """
+        if issues_created > 0:
+            logger.info(f"Issues processed: {issues_created} created/updated, {issues_skipped} skipped")
+        elif issues_skipped > 0:
+            logger.warning(f"Issues processed: 0 created, {issues_skipped} skipped (assets not found)")
+        else:
+            logger.debug("No issues processed")
+
+    def _finalize_scan_history(self, scan_history: regscale_models.ScanHistory) -> None:
+        """
+        Finalize scan history with error handling.
+
+        :param regscale_models.ScanHistory scan_history: Scan history to update
+        :return: None
+        :rtype: None
+        """
+        try:
+            if getattr(self, "enable_scan_history", True):
+                self._update_scan_history(scan_history)
+        except Exception:
+            self._update_scan_history(scan_history)
+
     def _ensure_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
         """
         Ensure an asset exists for the given finding.
@@ -733,14 +884,15 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             )
             assessments_created += created
 
-        logger.info(f"Successfully created {assessments_created} control assessments")
+        if assessments_created > 0:
+            logger.info(f"Successfully created {assessments_created} control assessments")
         passing_assessments = len([cid for cid in all_control_ids if cid not in self.failing_controls])
         failing_assessments = len([cid for cid in all_control_ids if cid in self.failing_controls])
         logger.info(f"Assessment breakdown: {passing_assessments} passing, {failing_assessments} failing")
-        logger.info(f"Control implementation mappings created: {len(self._impl_id_by_control)}")
+        logger.debug(f"Control implementation mappings created: {len(self._impl_id_by_control)}")
         if self._impl_id_by_control:
             logger.debug(f"Sample mappings: {dict(list(self._impl_id_by_control.items())[:5])}")
-        logger.info(f"Today's assessments by implementation: {len(self._assessment_by_impl_today)}")
+        logger.debug(f"Today's assessments by implementation: {len(self._assessment_by_impl_today)}")
         if self._assessment_by_impl_today:
             logger.debug(f"Sample assessment mappings: {dict(list(self._assessment_by_impl_today.items())[:5])}")
 

regscale/integrations/public/fedramp/fedramp_five.py

@@ -1063,7 +1063,7 @@ def _update_existing_control(
         existing_control,
         control_dict.get("status"),
         new_statement if new_statement else control_dict.get("statement"),
-        control_dict.get("responsibility"),
+        control_dict.get("origination"),
         primary_role if primary_role and isinstance(primary_role, dict) and primary_role.get("id") else None,
         parent_id,
     )
@@ -1087,14 +1087,93 @@ def _update_existing_control(
 
 def add_roles_to_control_implementation(implementation: ControlImplementation, roles: List[Dict]):
     """
-    Adds roles to a control implementation.
+    Updates roles for a control implementation by checking existing roles and adding/removing as appropriate.
+    This prevents duplicate roles on successive imports.
+    :param ControlImplementation implementation: The control implementation.
+    :param List[Dict] roles: The list of roles to set.
+    """
+    if not implementation or not implementation.id:
+        logger.warning("Control implementation is missing or has no ID, cannot update roles")
+        return
+
+    try:
+        # Get existing roles for this control implementation
+        from regscale.models.regscale_models.implementation_role import ImplementationRole
+
+        # Get existing roles
+        existing_roles = ImplementationRole.get_all_by_parent(
+            parent_id=implementation.id, parent_module=implementation._module_string
+        )
+        existing_role_ids = {role.roleId for role in existing_roles if role and role.roleId}
+
+        # Get target role IDs from the new roles list
+        target_role_ids = {role.get("id") for role in roles if isinstance(role, dict) and role.get("id")}
+
+        # Find roles to add (in target but not in existing)
+        roles_to_add = target_role_ids - existing_role_ids
+
+        # Find roles to remove (in existing but not in target)
+        roles_to_remove = existing_role_ids - target_role_ids
+
+        # Add new roles
+        for role_id in roles_to_add:
+            try:
+                implementation.add_role(role_id)
+                logger.debug(f"Added role {role_id} to control implementation {implementation.id}")
+            except Exception as e:
+                logger.warning(f"Failed to add role {role_id} to control implementation {implementation.id}: {e}")
+
+        # Remove roles that are no longer needed
+        _remove_roles_from_control_implementation(implementation, roles_to_remove, existing_roles)
+
+        if roles_to_add or roles_to_remove:
+            logger.info(
+                f"Updated roles for control implementation {implementation.id}: added {len(roles_to_add)}, removed {len(roles_to_remove)}"
+            )
+        else:
+            logger.debug(f"No role changes needed for control implementation {implementation.id}")
+
+    except Exception as e:
+        logger.error(f"Error updating roles for control implementation {implementation.id}: {e}")
+        # Fallback to old behavior if there's an error
+        _fallback_add_roles_to_control_implementation(implementation, roles)
+
+
+def _remove_roles_from_control_implementation(
+    implementation: ControlImplementation, roles_to_remove: set, existing_roles: List
+):
+    """
+    Removes roles that are no longer needed from a control implementation.
+    :param ControlImplementation implementation: The control implementation to remove roles from.
+    :param set roles_to_remove: Set of role IDs that should be removed.
+    :param List existing_roles: List of existing ImplementationRole objects.
+    """
+    for role_id in roles_to_remove:
+        try:
+            # Find the ImplementationRole record to delete
+            for existing_role in existing_roles:
+                if existing_role.roleId == role_id:
+                    existing_role.delete()
+                    logger.debug(f"Removed role {role_id} from control implementation {implementation.id}")
+                    break
+        except Exception as e:
+            logger.warning(f"Failed to remove role {role_id} from control implementation {implementation.id}: {e}")
+
+
+def _fallback_add_roles_to_control_implementation(implementation: ControlImplementation, roles: List[Dict]):
+    """
+    Fallback method for adding roles to a control implementation when the main method fails.
+    This uses the old behavior of simply adding roles without checking for duplicates.
     :param ControlImplementation implementation: The control implementation.
-    :param List[Dict] roles: The list of roles.
+    :param List[Dict] roles: The list of roles to add.
     """
-    if roles and len(roles) > 0 and implementation:
+    if roles and len(roles) > 0:
         for role in roles:
             if isinstance(role, dict) and role.get("id"):
-                implementation.add_role(role.get("id"))
+                try:
+                    implementation.add_role(role.get("id"))
+                except Exception as add_error:
+                    logger.warning(f"Failed to add role {role.get('id')}: {add_error}")
 
 
 def get_primary_and_supporting_roles(roles: List, parent_id: int) -> Tuple[List, Dict]:
@@ -1446,7 +1525,9 @@ def handle_matching_objectives(
         part_statement = f"{part.get('value', '')}" if not new_statement else new_statement
         statements_used.append(part_statement)
 
-        has_existing_obj = check_for_existing_objective(control_implementation, objective, status, part_statement)
+        has_existing_obj = check_for_existing_objective(
+            control_implementation, objective, status, part_statement, origination
+        )
         if has_existing_obj:
             continue
         duplicate = True if part_statement in statements_used else False
@@ -1467,6 +1548,7 @@ def check_for_existing_objective(
     objective: ControlObjective,
     status: Optional[str],
     part_statement: str,
+    origination: Optional[str] = None,
 ) -> bool:
     """
     Check for existing implementation objectives.
@@ -1474,6 +1556,7 @@ def check_for_existing_objective(
     :param ControlObjective objective: The control objective object.
     :param Optional[str] status: The status of the implementation.
     :param str part_statement: The part statement.
+    :param Optional[str] origination: The origination of the implementation.
     :return: True if an existing implementation objective is found, False otherwise.
     :rtype: bool
     """
@@ -1495,6 +1578,8 @@ def check_for_existing_objective(
                     status = status.value
                 existing_obj.status = status_map.get(status, status)
             existing_obj.statement = part_statement
+            if origination is not None:
+                existing_obj.responsibility = origination
             existing_obj.parentObjectiveId = objective.id
             existing_obj.save()
             return True
@@ -1755,7 +1840,7 @@ def update_existing_control(
     if not control.stepsToImplement and steps_to_implement:
         control.stepsToImplement = steps_to_implement
     control.implementation = state_text
-    control.responsibility = responsibility
+    control.responsibility = map_responsibility(responsibility)
     control.systemRoleId = primary_role.get("id") if primary_role and isinstance(primary_role, dict) else None
     # Clean statement
     # So, exclusion

regscale/integrations/scanner_integration.py

@@ -286,6 +286,8 @@ class IntegrationAsset:
 
     source_data: Optional[Dict[str, Any]] = None
     url: Optional[str] = None
+    software_function: Optional[str] = None
+    baseline_configuration: Optional[str] = None
     ports_and_protocols: List[Dict[str, Any]] = dataclasses.field(default_factory=list)
     software_inventory: List[Dict[str, Any]] = dataclasses.field(default_factory=list)
 
@@ -572,6 +574,11 @@ class ScannerIntegration(ABC):
 
     :param int plan_id: The ID of the security plan
     :param int tenant_id: The ID of the tenant, defaults to 1
+
+    Configuration options available via kwargs:
+    - suppress_asset_not_found_errors (bool): When True, suppresses "Asset not found" error messages
+      that are commonly logged when assets referenced in findings don't exist in RegScale.
+      This can help reduce log noise in environments with many missing assets.
     """
 
     stig_mapper = None
@@ -627,13 +634,18 @@ class ScannerIntegration(ABC):
     close_outdated_findings = True
     closed_count = 0
 
+    # Error suppression options
+    suppress_asset_not_found_errors = False
+
     def __init__(self, plan_id: int, tenant_id: int = 1, is_component: bool = False, **kwargs):
         """
         Initialize the ScannerIntegration.
 
         :param int plan_id: The ID of the security plan
         :param int tenant_id: The ID of the tenant, defaults to 1
+        :param bool is_component: Whether this is a component integration
         :param kwargs: Additional keyword arguments
+            - suppress_asset_not_found_errors (bool): If True, suppress "Asset not found" error messages
         """
         self.app = Application()
         self.alerted_assets: Set[str] = set()
@@ -642,6 +654,9 @@ class ScannerIntegration(ABC):
         self.plan_id: int = plan_id
         self.tenant_id: int = tenant_id
         self.is_component: bool = is_component
+
+        # Set configuration options from kwargs
+        self.suppress_asset_not_found_errors = kwargs.get("suppress_asset_not_found_errors", False)
         if self.is_component:
             self.component = regscale_models.Component.get_object(self.plan_id)
             self.parent_module: str = regscale_models.Component.get_module_string()
@@ -1138,6 +1153,8 @@ class ScannerIntegration(ABC):
             bAuthenticatedScan=asset.is_authenticated_scan,
             systemAdministratorId=asset.system_administrator_id,
             scanningTool=asset.scanning_tool,
+            softwareFunction=asset.software_function,
+            baselineConfiguration=asset.baseline_configuration,
         )
         if self.asset_identifier_field:
             setattr(new_asset, self.asset_identifier_field, asset.identifier)
@@ -2081,7 +2098,8 @@ class ScannerIntegration(ABC):
         asset = self.asset_map_by_identifier.get(identifier)
         if not asset and identifier not in self.alerted_assets:
             self.alerted_assets.add(identifier)
-            self.log_error("1. Asset not found for identifier %s", identifier)
+            if not getattr(self, "suppress_asset_not_found_errors", False):
+                self.log_error("1. Asset not found for identifier %s", identifier)
         return asset
 
     def get_issue_by_integration_finding_id(self, integration_finding_id: str) -> Optional[regscale_models.Issue]:
@@ -2106,7 +2124,8 @@ class ScannerIntegration(ABC):
         """
         logger.debug("Processing checklist %s", finding.external_id)
         if not (asset := self.get_asset_by_identifier(finding.asset_identifier)):
-            logger.error("2. Asset not found for identifier %s", finding.asset_identifier)
+            if not getattr(self, "suppress_asset_not_found_errors", False):
+                logger.error("2. Asset not found for identifier %s", finding.asset_identifier)
             return 0
 
         tool = regscale_models.ChecklistTool.STIGs
@@ -2333,7 +2352,8 @@ class ScannerIntegration(ABC):
         # Process checklist if applicable
         if self.type == ScannerIntegrationType.CHECKLIST:
             if not (asset := self.get_asset_by_identifier(finding.asset_identifier)):
-                logger.error("2. Asset not found for identifier %s", finding.asset_identifier)
+                if not getattr(self, "suppress_asset_not_found_errors", False):
+                    logger.error("2. Asset not found for identifier %s", finding.asset_identifier)
                 return
 
             tool = regscale_models.ChecklistTool.STIGs
@@ -2471,7 +2491,10 @@ class ScannerIntegration(ABC):
             return None
 
         if not asset:
-            logger.warning("VulnerabilityMapping Error: Asset not found for identifier %s", finding.asset_identifier)
+            if not getattr(self, "suppress_asset_not_found_errors", False):
+                logger.warning(
+                    "VulnerabilityMapping Error: Asset not found for identifier %s", finding.asset_identifier
+                )
             return None
 
         vulnerability = self.create_vulnerability_from_finding(finding, asset, scan_history)

regscale/models/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """standard python imports"""
-from regscale.models.app_models.click import regscale_id, regscale_module, regscale_ssp_id
+from regscale.models.app_models.click import regscale_id, regscale_module, regscale_ssp_id, ssp_or_component_id
 from .app_models import *
 from .integration_models import *
 from .regscale_models import *