regscale-cli 6.16.0.0__py3-none-any.whl

regscale/integrations/commercial/stig_mapper_integration/mapping_engine.py

@@ -0,0 +1,353 @@
+"""
+STIG Mapping Engine
+"""
+
+import json
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+from regscale.models import SecurityPlan
+from regscale.models.regscale_models import Asset, AssetMapping, Component
+
+logger = logging.getLogger(__name__)
+
+
+class StigMappingEngine:
+    """
+    A class to map assets to STIGs based on defined rules
+    """
+
+    comparator_functions = {
+        "equals": lambda a, b: a == b,
+        "contains": lambda a, b: b in a,
+        "notcontains": lambda a, b: b not in a,
+        "startswith": lambda a, b: a.startswith(b),
+        "notin": lambda a, b: b not in a,
+        "endswith": lambda a, b: a.endswith(b),
+        "notstartswith": lambda a, b: not a.startswith(b),
+        "notendswith": lambda a, b: not a.endswith(b),
+        "gt": lambda a, b: a > b,
+        "lt": lambda a, b: a < b,
+        "gte": lambda a, b: a >= b,
+        "lte": lambda a, b: a <= b,
+        "ne": lambda a, b: a != b,
+        "in": lambda a, b: a in b,
+        "nin": lambda a, b: a not in b,
+    }
+
+    def __init__(self, json_file: str):
+        self._mapping_cache = None
+        self._component_cache = None
+        self.rules = self.load_rules(json_file)
+        logger.info(f"Loaded {len(self.rules)} rules from {json_file}")
+        # Preprocess rules for faster access
+        self.stig_to_rules = {}
+        for rule in self.rules:
+            stig_name = rule.get("stig")
+            if stig_name not in self.stig_to_rules:
+                self.stig_to_rules[stig_name] = []
+            self.stig_to_rules[stig_name].append(rule.get("comparators", []))
+
+    @staticmethod
+    def load_rules(json_file: str) -> List[Dict[str, str]]:
+        """
+        Load rules from a JSON file
+
+        :param str json_file: The path to the JSON file
+        :return: A list of rules
+        :rtype: List[Dict[str, str]]
+        """
+        if not os.path.exists(json_file):
+            logger.error(f"File not found: {json_file}")
+            return []
+        try:
+            with open(json_file, "r") as file:
+                data = json.load(file)
+                return data.get("rules", [])
+        except json.JSONDecodeError as e:
+            logger.error(f"JSON decoding error in file {json_file}: {e}")
+        except Exception as e:
+            logger.error(f"Error loading rules from {json_file}: {e}")
+        return []
+
+    def evaluate_match(self, item: Any, comparators: List[Dict[str, str]]) -> bool:
+        """
+        Evaluates a single item (Asset or software inventory item) against multiple comparator rules
+
+        :param Any item: The asset or software inventory item
+        :param List[Dict[str, str]] comparators: List of comparators
+        :return: True if item meets all 'and' comparators or at least one 'or' comparator, otherwise False
+        :rtype: bool
+        """
+        # Separate comparators by their logical operator
+        and_comparators = [comp for comp in comparators if comp.get("logical_operator", "and").lower() == "and"]
+        or_comparators = [comp for comp in comparators if comp.get("logical_operator", "and").lower() == "or"]
+
+        # Evaluate 'and' comparators
+        if not all(self.evaluate_single_comparator(item, comp) for comp in and_comparators):
+            return False
+
+        # Evaluate 'or' comparators
+        if or_comparators:
+            return any(self.evaluate_single_comparator(item, comp) for comp in or_comparators)
+
+        # If all 'and' comparators passed and there are no 'or' comparators
+        return True
+
+    @staticmethod
+    def get_item_value(item: Any, property_name: Any) -> Optional[Any]:
+        """
+        Fetches the property value from the item, or None if not found
+
+        :param Any item: The asset or software inventory item
+        :param Any property_name: The property name
+        :return: The property value
+        :rtype: Optional[Any]
+        """
+        if isinstance(item, dict):
+            return item.get(property_name)
+        return getattr(item, property_name, None)
+
+    def evaluate_single_comparator(self, item: Any, comparator: dict) -> bool:
+        """
+        Evaluates a single comparator against the item
+
+        :param Any item: The asset or software inventory item
+        :param Dict[str, str] comparator: The comparator
+        :return: True if the item satisfies the comparator, otherwise False
+        :rtype: bool
+        """
+        property_name = comparator.get("property")
+        item_value = self.get_item_value(item, property_name)
+
+        if item_value is None:
+            return False
+
+        operator = comparator.get("comparator")
+        value = comparator.get("value")
+        comparator_func = StigMappingEngine.comparator_functions.get(operator)
+
+        return comparator_func(item_value, value) if comparator_func else False
+
+    @staticmethod
+    def find_matching_stigs(items: List[Dict[str, Any]], rules: List[Dict[str, Any]]) -> List[str]:
+        """
+        Checks a list of items (software inventory or assets) to see which STIGs they match
+
+        :param List[Dict[str, Any]] items: List of items (e.g., software inventory dictionaries)
+        :param List[Dict[str, Any]] rules: List of STIG rules, each containing a "stig" name and comparators
+        :return: List of matched STIG names
+        :rtype: List[str]
+        """
+        matched_stigs = []
+
+        for rule in rules:
+            stig_name = rule.get("stig")
+            comparators = rule.get("comparators", [])
+
+            # Track satisfaction of each comparator across all items
+            comparator_match = {i: False for i in range(len(comparators))}
+
+            # Go through each comparator and attempt to satisfy it with any item
+            for i, comparator in enumerate(comparators):
+                property_name = comparator.get("property")
+                value = comparator.get("value")
+                operator = comparator.get("comparator")
+
+                # Check if any item satisfies this comparator
+                for item in items:
+                    item_value = item.get(property_name)
+
+                    # Retrieve the comparison function
+                    comparator_func = StigMappingEngine.comparator_functions.get(operator)
+                    if comparator_func and comparator_func(item_value, value):
+                        comparator_match[i] = True
+                        break  # Move to the next comparator once a match is found
+
+            # Evaluate final match based on logical operators
+            if all(comparator_match.values()):
+                matched_stigs.append(stig_name)
+
+        return matched_stigs
+
+    @staticmethod
+    def asset_matches_comparators(asset: Asset, comparators: List[Dict[str, str]]) -> bool:
+        """
+        Determine if the asset matches the given comparators
+
+        :param Asset asset: An asset
+        :param List[Dict[str, str]] comparators: List of comparator dictionaries
+        :return: True if the asset matches the comparators, False otherwise
+        :rtype: bool
+        """
+        match_result = True
+
+        for comparator in comparators:
+            property_name = comparator.get("property")
+            if not hasattr(asset, property_name):
+                return False
+
+            operator = comparator.get("comparator")
+            comparator_func = StigMappingEngine.comparator_functions.get(operator)
+            if not comparator_func:
+                return False
+
+            value = comparator.get("value")
+            asset_value = getattr(asset, property_name)
+            comparison_result = comparator_func(asset_value, value)
+
+            logical_operator = comparator.get("logical_operator", "and").lower()
+
+            if logical_operator == "and":
+                match_result = match_result and comparison_result
+                if not match_result:
+                    return False
+            elif logical_operator == "or":
+                match_result = match_result or comparison_result
+            else:
+                logger.warning(f"Unknown logical operator: {logical_operator}")
+                return False
+
+        return match_result
+
+    def match_asset_to_stigs(
+        self, asset: Asset, ssp_id: int, software_inventory: Optional[List] = None
+    ) -> List[Component]:
+        """
+        Match an asset to STIG components based on rules
+
+        :param Asset asset: An asset
+        :param int ssp_id: The security plan ID
+        :param Optional[List] software_inventory: A list of software inventory
+        :return: A list of matching components
+        :rtype: List[Component]
+        """
+        if software_inventory is None:
+            software_inventory = []
+        if not self.rules:
+            return []
+
+        matching_components = []
+
+        # Ensure component cache is initialized
+        if self._component_cache is None:
+            self._component_cache = self.get_component_dict(ssp_id)
+
+        for stig_name, comparators_list in self.stig_to_rules.items():
+            component = self._component_cache.get(stig_name)
+            if not component:
+                continue
+
+            for comparators in comparators_list:
+                if self.asset_matches_comparators(asset, comparators):
+                    matching_components.append(component)
+                    break  # No need to check other comparators for this STIG
+
+        return matching_components
+
+    def map_stigs_to_assets(
+        self,
+        assets: List[Asset],
+        ssp_id: int,
+    ) -> List[AssetMapping]:
+        """
+        Map STIG components to assets based on rules
+
+        :param List[Asset] asset_list assets: A list of assets
+        :param List[Component] ssp_id: The security plan ID
+        :return: A list of asset mappings
+        :rtype: List[AssetMapping]
+        """
+        new_mappings = []
+
+        # Cache components to avoid redundant database queries
+        components = self.get_components(ssp_id)
+
+        # Build a mapping of existing mappings for quick lookup
+        existing_mappings = {}
+        for component in components:
+            mappings = AssetMapping.find_mappings(component_id=component.id)
+            existing_mappings[component.id] = {m.assetId for m in mappings}
+
+        for stig_name, comparators_list in self.stig_to_rules.items():
+            component = self._component_cache.get(stig_name)
+            if not component:
+                continue
+
+            component_existing_asset_ids = existing_mappings.get(component.id, set())
+
+            for asset in assets:
+                for comparators in comparators_list:
+                    if self.asset_matches_comparators(asset, comparators):
+                        if asset.id not in component_existing_asset_ids:
+                            mapping = AssetMapping(assetId=asset.id, componentId=component.id)
+                            new_mappings.append(mapping)
+                            component_existing_asset_ids.add(asset.id)
+                            logger.info(f"Mapping -> Asset ID: {asset.id}, Component ID: {component.id}")
+                        else:
+                            logger.info(
+                                f"Existing mapping found for Asset ID: {asset.id}, Component ID: {component.id}"
+                            )
+                        break  # No need to check other comparators for this asset and STIG
+
+        return new_mappings
+
+    def get_components(self, ssp_id: int) -> List[Component]:
+        """
+        Get all components for the given security plan
+
+        :param int ssp_id: The security plan ID
+        :return: A list of components
+        :rtype: List[Component]
+        """
+        if not hasattr(self, "_component_cache"):
+            components = Component.get_all_by_parent(parent_module=SecurityPlan.get_module_slug(), parent_id=ssp_id)
+            self._component_cache = {comp.title: comp for comp in components}
+        else:
+            components = self._component_cache.values()
+        return components
+
+    def get_component_dict(self, ssp_id: int) -> Dict[str, Component]:
+        """
+        Get a dictionary of components for the given security plan
+
+        :param int ssp_id: The security plan ID
+        :return: A dictionary of components
+        :rtype: Dict[str, Component]
+        """
+        if not hasattr(self, "_component_cache") or self._component_cache is None:
+            components = Component.get_all_by_parent(parent_module=SecurityPlan.get_module_slug(), parent_id=ssp_id)
+            self._component_cache = {comp.title: comp for comp in components}
+        return self._component_cache
+
+    def map_associated_stigs_to_asset(self, asset: Asset, ssp_id: int) -> List[AssetMapping]:
+        """
+        Map associated STIGs to an asset based on rules
+
+        :param Asset asset: An asset
+        :param int ssp_id: The security plan ID
+        :return: A list of asset mappings
+        :rtype: List[AssetMapping]
+        """
+        new_mappings = []
+        associated_components = self.match_asset_to_stigs(asset, ssp_id)
+
+        # Initialize or update the mapping cache
+        if not hasattr(self, "_mapping_cache") or self._mapping_cache is None:
+            mappings = AssetMapping.get_all_by_parent(parent_module=Asset.get_module_slug(), parent_id=asset.id)
+            self._mapping_cache = {m.componentId: m for m in mappings}
+
+        existing_component_ids = set(self._mapping_cache.keys())
+
+        for component in associated_components:
+            if component.id not in existing_component_ids:
+                mapping = AssetMapping(assetId=asset.id, componentId=component.id)
+                mapping.create()
+                new_mappings.append(mapping)
+                self._mapping_cache[component.id] = mapping
+                logger.debug(f"Created mapping for Asset ID: {asset.id}, Component ID: {component.id}")
+            else:
+                logger.debug(f"Mapping already exists for Asset ID: {asset.id}, Component ID: {component.id}")
+
+        return new_mappings

regscale/integrations/commercial/stigv2/ckl_parser.py

@@ -0,0 +1,349 @@
+"""
+This module is responsible for parsing STIG (Security Technical Implementation Guide) .ckl (Checklist) files.
+It provides functionality to extract relevant information from .ckl files for further processing and analysis.
+"""
+
+import io
+import logging
+from enum import Enum
+from pathlib import Path
+from typing import List, Dict, Optional, Any, Generator, Union
+
+import lxml.etree as et
+from lxml.etree import Element
+from pydantic import BaseModel, ConfigDict
+
+from regscale.core.utils import snakify
+
+logger = logging.getLogger("regscale")
+
+
+def stig_component_title(stig_title: str) -> str:
+    """
+    Extract the component title from the STIG title.
+
+    :param str stig_title: The STIG title.
+    :return: The component title.
+    :rtype: str
+    """
+    return stig_title.replace("  ", " ").strip()  # Ensure no double spaces are present
+
+
+def find_stig_files(directory: Path) -> List[Path]:
+    """
+    Finds all STIG (.ckl) files within the specified directory.
+
+    :param Path directory: Path to the directory where STIG files are located.
+    :return: A list of paths to STIG files.
+    :rtype: List[Path]
+    """
+    if str(directory).startswith("s3://"):
+        logger.info(f"Found S3 URI: {directory}")
+        return [directory]
+
+    if str(directory).endswith(".ckl"):
+        logger.info(f"Found CKL file: {directory}")
+        return [directory]
+
+    logger.info(f"Searching for STIG files in {directory}")
+    path = Path(directory)
+
+    if not path.is_dir():
+        logger.error(f"{directory} is not a valid directory")
+        return []
+
+    def is_hidden(filepath):
+        return any(part.startswith(".") for part in filepath.parts)
+
+    stig_files: list[Path] = [file for file in path.glob("**/*.ckl") if not is_hidden(file)]
+    logger.info(f"Found {len(stig_files)} STIG file(s)")
+    return stig_files
+
+
+class Asset(BaseModel):
+    """Data model for Asset with optional attributes."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=snakify)
+
+    role: Optional[str] = None
+    asset_type: Optional[str] = None
+    host_name: Optional[str] = None
+    host_ip: Optional[str] = None
+    host_mac: Optional[str] = None
+    host_fqdn: Optional[str] = None
+    tech_area: Optional[str] = None
+    target_key: Optional[str] = None
+    web_or_database: Optional[bool] = None
+    web_db_site: Optional[str] = None
+    web_db_instance: Optional[str] = None
+
+
+class STIGInfo(BaseModel):
+    """Data model for STIG Information with optional and required attributes."""
+
+    version: str
+    classification: str
+    customname: Optional[str] = None
+    stigid: str
+    description: Optional[str] = None
+    filename: Optional[str] = None
+    releaseinfo: str
+    title: str
+    uuid: str
+    notice: str
+    source: Optional[str] = None
+
+
+class VulnStatus(Enum):
+    """Enumeration for STIG vulnerability status."""
+
+    OPEN = "OPEN"
+    NOT_A_FINDING = "NOT A FINDING"
+    NOT_APPLICABLE = "NOT APPLICABLE"
+    COMPLETED = "COMPLETED"
+
+
+class Vuln(BaseModel):
+    """Data model for Vulnerability with optional and required attributes."""
+
+    vuln_num: str
+    severity: str
+    group_title: str
+    rule_id: str
+    rule_ver: str
+    rule_title: str
+    check_content: Optional[str] = None
+    fix_text: str
+    check_content_ref: Optional[str] = None
+    weight: str
+    stigref: Optional[str] = None
+    targetkey: Optional[str] = None
+    stig_uuid: str
+    vuln_discuss: Optional[str] = None
+    ia_controls: Optional[str] = None
+    class_: Optional[str] = None
+    cci_ref: list[str] = []
+    false_positives: Optional[str] = None
+    false_negatives: Optional[str] = None
+    documentable: Optional[str] = None
+    mitigations: Optional[str] = None
+    potential_impact: Optional[str] = None
+    third_party_tools: Optional[str] = None
+    mitigation_control: Optional[str] = None
+    responsibility: Optional[str] = None
+    security_override_guidance: Optional[str] = None
+    legacy_id: Optional[str] = None
+    status: Optional[str] = None
+    finding_details: Optional[str] = None
+    comments: Optional[str] = None
+    severity_override: Optional[str] = None
+    severity_justification: Optional[str] = None
+
+
+class STIG(BaseModel):
+    """Data model for STIG including STIG information and vulnerabilities."""
+
+    baseline: str
+    stig_info: STIGInfo
+    vulns: List[Vuln] = []
+
+    @property
+    def component_title(self) -> str:
+        """
+        Extract the component title from the STIG title.
+
+        :return: The component title.
+        :rtype: str
+        """
+        return stig_component_title(self.stig_info.title)
+
+
+class Checklist(BaseModel):
+    """Data model for Checklist including Asset and STIGs."""
+
+    assets: List[Asset] = []
+    stigs: List[STIG] = []
+
+
+def parse_element_to_dict(
+    element: Element,
+    key_tag: Optional[str] = None,
+    value_tag: Optional[str] = None,
+    list_fields: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """
+    This function parses an XML element into a dictionary. It can operate in two modes:
+    1. If key_tag and value_tag are provided, it searches for these specific tags within each child element
+       to construct key-value pairs for the dictionary.
+    2. If key_tag and value_tag are not provided, it converts all child elements of the given element
+       directly to a dictionary with child tag names as keys and their text content as values.
+
+    :param Element element: The XML element to parse.
+    :param Optional[str] key_tag: Optional; the tag name that contains the key for key-value parsing mode.
+    :param Optional[str] value_tag: Optional; the tag name that contains the value for key-value parsing mode.
+    :param Optional[List[str]] list_fields: Optional; list of fields that are expected to have lists back.
+    :return: A dictionary representation of the element.
+    :rtype: Dict[str, Any]
+    """
+    if list_fields is None:
+        list_fields = []
+
+    if key_tag and value_tag:
+        return _parse_key_value_pairs(element, key_tag, value_tag, list_fields)
+    else:
+        return _parse_direct_child_elements(element)
+
+
+def _parse_key_value_pairs(element: Element, key_tag: str, value_tag: str, list_fields: List[str]) -> Dict[str, Any]:
+    """
+    Parse key-value pairs from XML element based on provided key and value tags.
+
+    :param Element element: The XML element to parse.
+    :param str key_tag: The tag name that contains the key for key-value parsing mode.
+    :param str value_tag: The tag name that contains the value for key-value parsing mode.
+    :param List[str] list_fields: List of fields that are expected to have lists back.
+    :return: A dictionary of parsed key-value pairs.
+    :rtype: Dict[str, Any]
+    """
+    parsed_data: dict[str, Any] = {}
+    for child in element.findall(f".//{key_tag}"):
+        key = child.text
+        value_element = child.find(f"../{value_tag}")
+        value = getattr(value_element, "text", None)
+        if key:
+            key = snakify(key)
+            if key in list_fields:
+                parsed_data.setdefault(key, []).append(value) if value else None
+            else:
+                parsed_data[key] = value
+    return parsed_data
+
+
+def _parse_direct_child_elements(element: Element) -> Dict[str, Any]:
+    """
+    Parse direct child elements of an XML element into a dictionary.
+
+    :param Element element: The XML element to parse.
+    :return: A dictionary of direct child elements.
+    :rtype: Dict[str, Any]
+    """
+    result = {}
+    for child in element:
+        key = snakify(child.tag)
+        # Use empty string for None to ensure the field is included
+        value = child.text if child.text is not None else ""
+        result[key] = value
+    return result
+
+
+def parse_checklist(file_path: Union[str, Path]) -> Checklist:
+    """
+    Main function to parse a checklist from an XML file or S3 object using lxml for parsing.
+
+    :param Union[str, Path] file_path: The path to the XML file or S3 object.
+    :raises ValueError: If the ASSET element is not found in the XML
+    :return: Checklist object
+    :rtype: Checklist
+    """
+    try:
+        import boto3
+
+        if isinstance(file_path, str) and file_path.startswith("s3://"):
+            # Handle S3 path
+            s3_parts = file_path[5:].split("/", 1)
+            bucket = s3_parts[0]
+            key = s3_parts[1]
+
+            s3 = boto3.client("s3")
+            response = s3.get_object(Bucket=bucket, Key=key)
+            content = response["Body"].read()
+
+            tree = et.parse(io.BytesIO(content))
+            root = tree.getroot()
+            file_name = key.split("/")[-1]
+        else:
+            # Handle local file path
+            xml_path = Path(file_path)
+            tree = et.parse(xml_path)
+            root = tree.getroot()
+            file_name = xml_path.name
+
+        # Parse the Assets
+        asset_elem = root.find("ASSET")
+        if asset_elem is None:
+            raise ValueError("ASSET element not found in XML.")
+        assets = [Asset(**parse_element_to_dict(asset_elem))]
+
+        stigs = [
+            STIG(
+                baseline=file_name.split(".")[0].split("-")[0].strip(),
+                stig_info=STIGInfo(**parse_element_to_dict(istig_elem, key_tag="SID_NAME", value_tag="SID_DATA")),
+                vulns=[
+                    Vuln(
+                        **parse_element_to_dict(
+                            vuln_elem,
+                            key_tag="VULN_ATTRIBUTE",
+                            value_tag="ATTRIBUTE_DATA",
+                            list_fields=[
+                                "cci_ref",
+                            ],
+                        ),
+                        status=vuln_elem.findtext("STATUS"),
+                        finding_details=vuln_elem.findtext("FINDING_DETAILS"),
+                        comments=vuln_elem.findtext("COMMENTS"),
+                        severity_override=vuln_elem.findtext("SEVERITY_OVERRIDE"),
+                        severity_justification=vuln_elem.findtext("SEVERITY_JUSTIFICATION"),
+                    )
+                    for vuln_elem in istig_elem.findall("VULN")
+                ],
+            )
+            for istig_elem in root.findall(".//iSTIG")
+            if istig_elem.find("STIG_INFO") is not None
+        ]
+
+        return Checklist(assets=assets, stigs=stigs)
+    except Exception as e:
+        logger.error(f"Error parsing {file_path}: {e}")
+        raise e
+
+
+def get_components_from_checklist(
+    checklist: Checklist,
+) -> Generator[dict[str, str], None, None]:
+    """
+    Extract the component titles from the checklist using a generator expression.
+
+    :param Checklist checklist: The checklist object.
+    :return: A generator of component titles.
+    :rtype: Generator[dict[str, str], None, None]
+    """
+    return ({stig.stig_info.stigid: stig_component_title(stig.stig_info.title)} for stig in checklist.stigs)
+
+
+def get_all_components_from_checklists(checklists: List[Checklist]) -> dict[str, str]:
+    """
+    Extract the component titles from a list of checklists using a generator expression.
+
+    :param List[Checklist] checklists: The list of checklist objects.
+    :return: A dictionary of unique component titles.
+    :rtype: dict[str, str]
+    """
+    components: dict[str, str] = {}
+    for checklist in checklists:
+        # Collect all components from the generator into a single dictionary
+        components.update({k: v for d in get_components_from_checklist(checklist) for k, v in d.items()})
+    return components
+
+
+def get_all_assets_from_checklists(checklists: List[Checklist]) -> List[Asset]:
+    """
+    Extract the assets from a list of checklists.
+
+    :param List[Checklist] checklists: The list of checklist objects.
+    :return: A list of unique assets.
+    :rtype: List[Asset]
+    """
+    assets = set()
+    for checklist in checklists:
+        assets.update(checklist.assets)
+    return list(assets)