regscale-cli 6.27.1.0__py3-none-any.whl → 6.27.3.0__py3-none-any.whl

regscale/integrations/public/cci_importer.py

@@ -6,10 +6,11 @@ import xml.etree.ElementTree as ET
 from typing import Dict, List, Optional, Tuple
 
 import click
+from rich.progress import Progress, TaskID
 
 from regscale.core.app.application import Application
 from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit
-from regscale.models.regscale_models import Catalog, SecurityControl, CCI
+from regscale.models.regscale_models import Catalog, SecurityControl, CCI, ControlObjective
 
 logger = logging.getLogger("regscale")
 
@@ -30,6 +31,7 @@ class CCIImporter:
         """
         self.xml_data = xml_data
         self.normalized_cci: Dict[str, List[Dict]] = {}
+        self.cci_grouped_by_index: Dict[str, str] = {}
         self.verbose = verbose
         self.reference_version = version
         self._user_context: Optional[Tuple[Optional[str], int]] = None
@@ -46,6 +48,222 @@ class CCIImporter:
         parts = ref_index.strip().split()
         return parts[0] if parts else ""
 
+    @staticmethod
+    def format_index(index: str) -> str:
+        """
+        Format index according to ControlObjective matching requirements.
+
+        Examples:
+            'AC-1 a 1' -> 'AC-1(a)(1)'
+            'IA-13 (03) (a)' -> 'IA-13(03)(a)'
+            'AC-1 a 1 (a)' -> 'AC-1(a)(1)(a)'
+
+        :param str index: Raw index string from XML
+        :return: Formatted index string
+        :rtype: str
+        """
+        import re
+
+        index = index.strip()
+
+        # Pattern: match either (text) or non-whitespace text
+        pattern = r"\([^)]+\)|[^\s()]+"
+        parts = re.findall(pattern, index)
+
+        if len(parts) <= 1:
+            return parts[0] if parts else index
+
+        # First part is the base control (e.g., 'AC-1', 'IA-13')
+        result = parts[0]
+
+        # Process remaining parts
+        for part in parts[1:]:
+            if part.startswith("("):
+                # Already has parentheses, just append
+                result += part
+            else:
+                # Need to add parentheses
+                result += f"({part})"
+
+        return result
+
+    @staticmethod
+    def parse_objective_id(objective_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Parse an objective otherId to extract control base and part.
+
+        Supports both NIST 800-53 Revision 4 and 5 formats:
+
+        Revision 5 Examples:
+            "ac-1_smt.a" -> ("AC-1", "a")
+            "ac-2.3_smt.a" -> ("AC-2(3)", "a")
+            "au-10.1_smt.a" -> ("AU-10(1)", "a")
+            "ac-2.4_smt" -> ("AC-2(4)", None)
+
+        Revision 4 Examples:
+            "ac-1_smt.a.1" -> ("AC-1", "a")
+            "ac-1_smt.b.2" -> ("AC-1", "b")
+            "ac-2.3_smt.d" -> ("AC-2(3)", "d")
+
+        :param str objective_id: Objective otherId value
+        :return: Tuple of (control_base, part_letter or None)
+        :rtype: Tuple[Optional[str], Optional[str]]
+        """
+        import re
+
+        # Pattern 1: xx-nn[.nn]_smt.x[.nn] (with part letter, optional subpart for rev 4)
+        # Matches: ac-1_smt.a, ac-1_smt.a.1, ac-2.3_smt.d
+        match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt\.([a-z]+)(?:\.\d+)?$", objective_id.lower())
+
+        if match:
+            family = match.group(1).upper()
+            control_num = match.group(2)
+            enhancement = match.group(3)
+            part = match.group(4)
+        else:
+            # Pattern 2: xx-nn[.nn]_smt[.x][.nn] (without clear part letter, or just enhancement)
+            # Matches: ac-2.4_smt, ac-1_smt
+            match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt(?:\.[a-z]+)?(?:\.\d+)?$", objective_id.lower())
+            if not match:
+                return None, None
+
+            family = match.group(1).upper()
+            control_num = match.group(2)
+            enhancement = match.group(3)
+            part = None
+
+        if enhancement:
+            # Enhancement like AC-2(3)
+            control_base = f"{family}-{control_num}({enhancement})"
+        else:
+            # Base control like AC-1
+            control_base = f"{family}-{control_num}"
+
+        return control_base, part
+
+    @staticmethod
+    def find_matching_ccis(control_base: str, part: Optional[str], cci_map: Dict[str, str]) -> List[str]:
+        """
+        Find all CCI IDs that match the control base and part.
+
+        Examples:
+            control_base="AC-1", part="a" matches:
+                - AC-1(a)(1)(a)
+                - AC-1(a)(1)(b)
+                - AC-1(a)(2)
+                - AC-1(a)
+
+        :param str control_base: Control identifier (e.g., "AC-1", "AC-2(3)")
+        :param Optional[str] part: Part letter (e.g., "a", "b") or None for enhancements
+        :param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
+        :return: List of CCI ID strings (comma-separated)
+        :rtype: List[str]
+        """
+        matching_ccis = []
+
+        for index, cci_ids in cci_map.items():
+            # Check if index starts with control_base
+            if index.startswith(control_base):
+                # Extract the part after control_base
+                remainder = index[len(control_base) :]
+
+                # For enhancements without parts, only match exact control (no remainder)
+                # And Check if the remainder starts with (part)
+                if (part is None and remainder == "") or remainder.startswith(f"({part})") or remainder == f"({part})":
+                    matching_ccis.append(cci_ids)
+
+        return matching_ccis
+
+    def find_matching_ccis_by_name(self, control_base: str, name: str, cci_map: Dict[str, str]) -> List[str]:
+        """
+        Find CCIs by matching control base and objective name field.
+        Fallback method when otherId matching fails.
+
+        Supports both NIST 800-53 Revision 4 and 5 label formats:
+
+        Revision 5 Examples:
+            control_base="AC-2(4)", name="AC-2(4)" -> matches AC-2(4)
+            control_base="AC-1", name="a" -> matches AC-1(a), AC-1(a)(1), etc.
+
+        Revision 4 Examples:
+            control_base="AC-1", name="a.1." -> matches AC-1(a), AC-1(a)(1), etc.
+            control_base="AC-1", name="b.2." -> matches AC-1(b), AC-1(b)(2), etc.
+
+        :param str control_base: Control identifier
+        :param str name: Objective name field
+        :param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
+        :return: List of CCI ID strings (comma-separated)
+        :rtype: List[str]
+        """
+        import re
+
+        matching_ccis = []
+
+        # Remove trailing period and whitespace from name for matching
+        clean_name = name.strip().rstrip(".").strip()
+
+        # If name exactly matches control_base, match that control
+        if clean_name == control_base:
+            if control_base in cci_map:
+                matching_ccis.append(cci_map[control_base])
+            return matching_ccis
+
+        # Try to extract part letter from different formats
+        part = None
+
+        # Check if name is a single letter (Revision 5 format: "a", "b")
+        if len(clean_name) == 1 and clean_name.isalpha():
+            part = clean_name.lower()
+        elif match := re.match(r"^([a-z])\.\d+$", clean_name.lower()):
+            part = match.group(1)
+
+        # If we extracted a part letter, try matching
+        if part:
+            matching_ccis = self._extract_part_letter(cci_map, control_base, part, matching_ccis)
+
+        return matching_ccis
+
+    @staticmethod
+    def _extract_part_letter(
+        cci_map: Dict[str, str], control_base: str, part: str, matching_ccis: List[str]
+    ) -> List[str]:
+        """
+        Extract the part letter from the name.
+
+        :param Dict[str, str] cci_map: The map of CCI IDs to their indices.
+        :param str control_base: The control base to match against.
+        :param str part: The part letter to match against.
+        :param List[str] matching_ccis: The list of matching CCI IDs.
+        :rtype: List[str]
+        """
+        for index, cci_ids in cci_map.items():
+            if index.startswith(control_base):
+                remainder = index[len(control_base) :]
+                if remainder.startswith(f"({part})") or remainder == f"({part})":
+                    matching_ccis.append(cci_ids)
+        return matching_ccis
+
+    @staticmethod
+    def ccis_already_present(current_other_id: str, new_cci_ids: str) -> bool:
+        """
+        Check if any of the new CCI IDs are already present in current otherId.
+        Prevents duplicate CCI mappings.
+
+        :param str current_other_id: Current otherId value
+        :param str new_cci_ids: Comma-separated string of CCI IDs to add
+        :return: True if any CCIs are already present
+        :rtype: bool
+        """
+        if not current_other_id:
+            return False
+
+        # Extract individual CCI IDs from both strings
+        existing_ccis: set[str] = {cci.strip() for cci in current_other_id.split(",") if cci.strip().startswith("CCI-")}
+        new_ccis: set[str] = {cci.strip() for cci in new_cci_ids.split(",") if cci.strip().startswith("CCI-")}
+
+        # Check if there's any overlap
+        return bool(existing_ccis & new_ccis)
+
     @staticmethod
     def _extract_cci_data(cci_item: ET.Element) -> Tuple[Optional[str], str]:
         """
@@ -104,20 +322,49 @@ class CCIImporter:
 
     def parse_cci(self) -> None:
         """
-        Parse CCI items from XML and normalize them, mapping to parent control only.
+        Parse CCI items from XML and create both mapping structures.
+
+        Creates:
+        - normalized_cci: Dict[control_id, List[Dict]] - for SecurityControl mapping
+        - cci_grouped_by_index: Dict[formatted_index, str] - for ControlObjective mapping
 
         :rtype: None
         """
         if self.verbose:
             logger.info("Parsing CCI items from XML...")
 
+        # Track all CCI items with formatted indices for objective mapping
+        from collections import defaultdict
+
+        temp_grouped: Dict[str, List[str]] = defaultdict(list)
+
         for cci_item in self.xml_data.findall(".//{http://iase.disa.mil/cci}cci_item"):
             cci_id, definition = self._extract_cci_data(cci_item)
             if not cci_id:
                 continue
 
             references = cci_item.findall(".//{http://iase.disa.mil/cci}reference")
-            self._process_references(references, cci_id, definition)
+
+            for ref in references:
+                if not self._is_valid_reference(ref):
+                    continue
+
+                ref_index = ref.get("index")
+                if ref_index:
+                    # Existing: simple control ID for SecurityControl mapping
+                    main_control = self._parse_control_id(ref_index)
+                    self._add_cci_to_control(main_control, cci_id, definition)
+
+                    # NEW: formatted index for ControlObjective mapping
+                    formatted_index = self.format_index(ref_index)
+                    temp_grouped[formatted_index].append(cci_id)
+
+        # Convert to comma-separated format
+        self.cci_grouped_by_index = {index: ", ".join(cci_list) for index, cci_list in temp_grouped.items()}
+
+        if self.verbose:
+            logger.info(f"Created {len(self.normalized_cci)} control mappings")
+            logger.info(f"Created {len(self.cci_grouped_by_index)} formatted index mappings")
 
     @staticmethod
     def _get_catalog(catalog_id: int) -> Catalog:
@@ -324,9 +571,116 @@ class CCIImporter:
             "created": created_count,
             "updated": updated_count,
             "skipped": skipped_count,
-            "total_processed": sum(created_count + updated_count + skipped_count),
+            "total_processed": len(self.normalized_cci),
+        }
+
+    def map_to_control_objectives(self, catalog_id: int = 1) -> Dict[str, int]:
+        """
+        Map grouped CCI data to control objectives in the database.
+        Updates the otherId field of existing ControlObjective records.
+
+        :param int catalog_id: ID of the catalog containing control objectives (default: 1)
+        :return: Dictionary with operation statistics
+        :rtype: Dict[str, int]
+        """
+        if self.verbose:
+            logger.info("Mapping CCI data to control objectives...")
+
+        # Fetch all objectives for the catalog
+        objectives: List[ControlObjective] = ControlObjective.get_by_catalog(catalog_id=catalog_id)
+
+        if self.verbose:
+            logger.info(f"Found {len(objectives)} objectives in catalog {catalog_id}")
+
+        objectives_updated = 0
+        objectives_skipped = 0
+        objectives_not_found = 0
+
+        with create_progress_object() as progress:
+            logger.info(f"Processing {len(objectives)} objectives...")
+            task = progress.add_task("Mapping CCIs to objectives...", total=len(objectives))
+
+            for obj in objectives:
+                objective_id = obj.otherId
+
+                # Skip objectives without proper otherId
+                if not objective_id or "_smt" not in objective_id:
+                    objectives_not_found += 1
+                    progress.update(task, advance=1)
+                    continue
+
+                # Extract just the objective ID part (before any CCIs)
+                # Format: "ac-1_smt.a" or "ac-1_smt.a, CCI-000001, CCI-000002"
+                objective_id_parts = objective_id.split(",")
+                base_objective_id = objective_id_parts[0].strip()
+
+                # Parse the objective ID
+                control_base, part = self.parse_objective_id(base_objective_id)
+
+                if not control_base:
+                    objectives_not_found += 1
+                    progress.update(task, advance=1)
+                    continue
+
+                # Find matching CCIs by otherId
+                matching_ccis = self.find_matching_ccis(control_base, part, self.cci_grouped_by_index)
+
+                # Fallback: try matching by name
+                if not matching_ccis and obj.name:
+                    matching_ccis = self.find_matching_ccis_by_name(control_base, obj.name, self.cci_grouped_by_index)
+
+                if matching_ccis:
+                    skipped_count, updated_count = self._handle_matching_ccis(
+                        control_objective=obj,
+                        matching_ccis=matching_ccis,
+                        base_objective_id=base_objective_id,
+                    )
+                    objectives_skipped += skipped_count
+                    objectives_updated += updated_count
+                else:
+                    objectives_not_found += 1
+
+                progress.update(task, advance=1)
+
+        return {
+            "updated": objectives_updated,
+            "skipped": objectives_skipped,
+            "not_found": objectives_not_found,
+            "total_processed": len(objectives),
         }
 
+    def _handle_matching_ccis(
+        self,
+        control_objective: ControlObjective,
+        matching_ccis: List[str],
+        base_objective_id: str,
+    ) -> Tuple[int, int]:
+        """
+        Handle matching CCIs.
+
+        :param ControlObjective control_objective: ControlObjective instance
+        :param List[str] matching_ccis: List of matching CCI IDs
+        :param str base_objective_id: Base objective ID
+        :return: Tuple of (number of objectives skipped, number of objectives updated)
+        :rtype: Tuple[int, int]
+        """
+        # Combine all CCI IDs
+        all_cci_ids = ", ".join(matching_ccis)
+
+        # Check for duplicates
+        if self.ccis_already_present(control_objective.otherId, all_cci_ids):
+            if self.verbose:
+                logger.info(f"Skipping {base_objective_id} - CCIs already present")
+            return 1, 0
+
+        # Update the otherId field
+        control_objective.otherId = f"{control_objective.otherId}, {all_cci_ids}"
+        control_objective.save()
+
+        if self.verbose:
+            logger.info(f"Updated {base_objective_id} with {all_cci_ids}")
+        return 0, 1
+
     def get_normalized_cci(self) -> Dict[str, List[Dict]]:
         """
         Get the normalized CCI data.
@@ -385,7 +739,25 @@ def _display_results(stats: Dict[str, int]) -> None:
     )
 
 
-def _process_cci_import(importer: CCIImporter, dry_run: bool, verbose: bool, catalog_id: int) -> None:
+def _display_objective_results(stats: Dict[str, int]) -> None:
+    """
+    Display control objective mapping results.
+
+    :param Dict[str, int] stats: Dictionary with operation statistics
+    :rtype: None
+    """
+    logger.info(
+        f"[green]\nControl objective operations completed:"
+        f"[green]\n  - Updated: {stats['updated']}"
+        f"[green]\n  - Skipped: {stats['skipped']}"
+        f"[green]\n  - Not found: {stats['not_found']}"
+        f"[green]\n  - Total processed: {stats['total_processed']}",
+    )
+
+
+def _process_cci_import(
+    importer: CCIImporter, dry_run: bool, verbose: bool, catalog_id: int, disable_objectives: bool = False
+) -> None:
     """
     Process CCI import with optional database operations.
 
@@ -393,6 +765,7 @@ def _process_cci_import(importer: CCIImporter, dry_run: bool, verbose: bool, cat
     :param bool dry_run: Whether to skip database operations
     :param bool verbose: Whether to display verbose output
     :param int catalog_id: ID of the catalog containing security controls
+    :param bool disable_objectives: Whether to disable mapping to control objectives
     :rtype: None
     """
     importer.parse_cci()
@@ -404,8 +777,15 @@ def _process_cci_import(importer: CCIImporter, dry_run: bool, verbose: bool, cat
         _display_verbose_output(normalized_data)
 
     if not dry_run:
+        # Map to SecurityControl (existing functionality)
         stats = importer.map_to_security_controls(catalog_id)
         _display_results(stats)
+
+        # Map to ControlObjective (new functionality)
+        if not disable_objectives:
+            logger.info("\n[cyan]Mapping CCIs to control objectives...[/cyan]")
+            obj_stats = importer.map_to_control_objectives(catalog_id)
+            _display_objective_results(obj_stats)
     else:
         logger.info("\n[yellow]DRY RUN MODE: No database changes were made[/yellow]")
 
@@ -422,10 +802,21 @@ def _process_cci_import(importer: CCIImporter, dry_run: bool, verbose: bool, cat
 @click.option(
     "--catalog-id", "-c", type=click.INT, default=1, help="ID of the catalog containing security controls (default: 1)"
 )
-def cci_importer(xml_file: str, dry_run: bool, verbose: bool, nist_version: str, catalog_id: int) -> None:
-    """Import CCI data from XML files and map to security controls.
+@click.option(
+    "--disable-objectives",
+    "-o",
+    is_flag=True,
+    help="Disable mapping CCIs to control objectives (updates otherId field)",
+)
+def cci_importer(
+    xml_file: str, dry_run: bool, verbose: bool, nist_version: str, catalog_id: int, disable_objectives: bool
+) -> None:
+    """Import CCI data from XML files and map to security controls and/or objectives.
+
+    By default, maps CCIs to SecurityControl entities. Use --disable-objectives flag
+    to also update ControlObjective.otherId fields with CCI mappings.
 
-    If no XML file is specified, defaults to 'artifacts/U_CCI_List.xml' in the project directory.
+    If no XML file is specified, defaults to packaged CCI_List.xml.
     """
 
     try:
@@ -438,6 +829,6 @@ def cci_importer(xml_file: str, dry_run: bool, verbose: bool, nist_version: str,
             xml_file = str(cci_path)
         root = _load_xml_file(xml_file)
         importer = CCIImporter(root, version=nist_version, verbose=verbose)
-        _process_cci_import(importer, dry_run, verbose, catalog_id)
+        _process_cci_import(importer, dry_run, verbose, catalog_id, disable_objectives)
     except Exception as e:
         error_and_exit(f"Unexpected error: {e}")