regscale-cli 6.16.0.0__py3-none-any.whl

regscale/core/app/utils/catalog_utils/update_plans.py

@@ -0,0 +1,334 @@
+import csv
+import datetime
+import uuid
+from typing import Dict, List, Any, Optional
+
+from regscale.core.app import create_logger
+from regscale.core.app.utils.catalog_utils.common import parentheses_to_dot
+from regscale.models import regscale_models as rm, ControlImplementationStatus
+
+logger = create_logger()
+# Define a list to hold log entries for CSV output
+log_entries: List[Dict[str, str]] = []
+
+
+def _log_change(action: str, model: str, message: str, changes: Optional[Dict[str, Any]] = None):
+    """
+    Log changes to both the logger and a list for later CSV output.
+    :param str action: The type of action (e.g., 'Add', 'Update', 'Remove')
+    :param str model: The model being changed (e.g., 'Control', 'Objective', 'Parameter')
+    :param str message: The message to log
+    :param Optional[Dict[str, Any]] changes: Optional changes to log
+    """
+    logger.info(message)
+    log_entries.append({"action": action, "model": model, "message": message, "changes": str(changes)})
+
+
+def _write_log_to_csv(filename: str):
+    """
+    Write all log entries to a CSV file.
+
+    :param str filename: The filename for the CSV output
+    """
+    entry_count = len(log_entries)
+    if entry_count > 0:
+        logger.info(f"Writing {entry_count} changes to {filename}")
+        with open(filename, "w", newline="") as csvfile:
+            fieldnames = ["action", "model", "message", "changes"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+            for entry in log_entries:
+                writer.writerow(entry)
+        logger.info(f"Successfully wrote all entries to {filename}")
+    else:
+        logger.info("No changes to write to CSV")
+
+
+def _log_summary():
+    """
+    Log statistics about the entries, summarized by action and model.
+    """
+    from collections import defaultdict
+    from rich.console import Console
+
+    console = Console()
+    summary: defaultdict[Any, int] = defaultdict(int)
+    for entry in log_entries:
+        key = (entry["action"], entry["model"])
+        summary[key] += 1
+
+    # Print statistics in a table-like format in green color
+    console.print("Summary of Changes:", style="green")
+    console.print(f"{'Action':<20} {'Model':<20} {'Count':<10}", style="green")
+    console.print("-" * 50, style="green")
+    for (action, model), count in summary.items():
+        console.print(f"{action:<20} {model:<20} {count:<10}", style="green")
+
+
+def sync_controls(catalog_controls: Dict[str, rm.SecurityControl], ssp_id: int, dry_run: bool = False):
+    """
+    Sync controls from a catalog to an SSP
+
+    :param Dict[str, rm.SecurityControl] catalog_controls: List of controls from the catalog
+    :param int ssp_id: SSP ID
+    :param bool dry_run: Dry run flag
+    """
+    logger.info(f"Syncing controls from catalog to SSP: {ssp_id}")
+
+    catalog_id_to_label = {x.id: x.controlId for x in catalog_controls.values()}
+
+    ssp = rm.SecurityPlan.get_object(object_id=ssp_id)
+    if not ssp:
+        logger.warning("SSP not found")
+        exit(1)
+
+    ssp_controls = {
+        parentheses_to_dot(catalog_id_to_label[x.controlID]): x
+        for x in rm.ControlImplementation.get_all_by_parent(
+            parent_id=ssp.id, parent_module=rm.SecurityPlan.get_module_string()
+        )
+        if x.controlID in catalog_id_to_label
+    }
+
+    # Add or update controls
+    for control_id, control in catalog_controls.items():
+        sync_control(control, control_id, dry_run, ssp, ssp_controls)
+
+    # Remove controls not in the catalog
+    for ssp_control_id, ssp_control in ssp_controls.items():
+        if ssp_control.controlID not in catalog_id_to_label:
+            _log_change("Remove", "Control", f"Removing control {ssp_control.controlID} from SSP {ssp.systemName}")
+            if not dry_run:
+                ssp_control.delete()
+
+
+def sync_control(
+    control: rm.Control,
+    control_id: str,
+    dry_run: bool,
+    ssp: rm.SecurityPlan,
+    ssp_controls: Dict[str, rm.ControlImplementation],
+):
+    """
+    Sync a control from a catalog to an SSP
+
+    :param rm.Control control: The control from the catalog
+    :param str control_id: The control ID
+    :param bool dry_run: Dry run flag
+    :param rm.SecurityPlan ssp: The SSP
+    :param Dict[str, rm.ControlImplementation] ssp_controls: The controls in the SSP
+    """
+    logger.debug(f"Syncing control {control_id}")
+    ssp_control = ssp_controls.get(parentheses_to_dot(control_id))
+    if ssp_control is None:
+        # Create a new control in the SSP
+        print(f"Control Owner ID: {ssp.createdById}")
+        print(f"API Handler User ID: {rm.ControlImplementation.get_user_id()}")
+        ssp_control = rm.ControlImplementation(
+            controlOwnerId=ssp.systemOwnerId or ssp.createdById or rm.ControlImplementation.get_user_id(),
+            status=ControlImplementationStatus.NotImplemented,
+            controlID=control.id,
+            parentId=ssp.id,
+            parentModule=ssp.get_module_string(),
+        )
+        _log_change(
+            "Add",
+            "Control",
+            f"Adding control {control.controlId} to SSP {ssp.systemName}",
+            changes=ssp_control.show_changes(),
+        )
+        if not dry_run:
+            ssp_control = ssp_control.create()
+    else:
+        # Update the existing control in the SSP
+        if ssp_control.controlOwnerId in ["", None]:
+            ssp_control.controlOwnerId = ssp.systemOwnerId or ssp.createdById or rm.ControlImplementation.get_user_id()
+        if ssp_control.has_changed():
+            _log_change(
+                "Update",
+                "Control",
+                f"Updating control {control.controlId} in SSP {ssp.systemName}",
+                changes=ssp_control.show_changes(),
+            )
+            if not dry_run:
+                ssp_control.save()
+    sync_objectives(control, dry_run, ssp_control)
+    sync_parameters(control, dry_run, ssp_control)
+
+
+def sync_objectives(control: rm.SecurityControl, dry_run: bool, ssp_control: rm.Control):
+    """
+    Sync objectives for a control
+
+    :param rm.SecurityControl control: The control
+    :param bool dry_run: Dry run flag
+    :param rm.Control ssp_control: The SSP control
+    """
+    ssp_objectives: Dict[int, rm.ImplementationObjective] = {  # type: ignore
+        x.objectiveId: x
+        for x in rm.ImplementationObjective.get_all_by_parent(
+            parent_id=ssp_control.id, parent_module=rm.ControlImplementation.get_module_string()
+        )
+    }
+    control_objectives: Dict[int, rm.ControlObjective] = {  # type: ignore
+        x.id: x
+        for x in rm.ControlObjective.get_all_by_parent(
+            parent_id=control.id, parent_module=rm.SecurityControl.get_module_string()
+        )
+    }
+
+    # Remove objectives that are not in the catalog
+    for objective_id in ssp_objectives.keys():
+        if objective_id not in control_objectives.keys():
+            _log_change(
+                "Remove",
+                "Objective",
+                f"Removing objective {objective_id} from control {control.controlId}",
+            )
+            if not dry_run:
+                ssp_objectives[objective_id].delete()
+
+
+def sync_parameters(control: rm.SecurityControl, dry_run: bool, ssp_control: rm.ControlImplementation):
+    """
+    Sync parameters for a control
+
+    :param rm.SecurityControl control: The control
+    :param bool dry_run: Dry run flag
+    :param rm.ControlImplementation ssp_control: The SSP control
+    """
+    ssp_parameters = get_ssp_parameters(ssp_control)
+    control_parameters = get_control_parameters(control)
+
+    # Add or update parameters
+    add_or_update_parameters(control_parameters, ssp_parameters, control, dry_run, ssp_control)
+
+    # Remove parameters that are not in the catalog
+    remove_unused_parameters(ssp_parameters, control_parameters, control, dry_run)
+
+
+def get_ssp_parameters(ssp_control):
+    return {
+        x.parentParameterId: x
+        for x in rm.Parameter.get_all_by_parent(
+            parent_id=ssp_control.id, parent_module=rm.ControlImplementation.get_module_string()
+        )
+    }
+
+
+def get_control_parameters(control):
+    return {
+        x.id: x
+        for x in rm.ControlParameter.get_all_by_parent(
+            parent_id=control.id, parent_module=rm.SecurityControl.get_module_string()
+        )
+    }
+
+
+def add_or_update_parameters(control_parameters, ssp_parameters, control, dry_run, ssp_control):
+    for control_parameter in control_parameters.values():
+        if control_parameter.id in ssp_parameters:
+            update_parameter(ssp_parameters[control_parameter.id], control_parameter, control, dry_run)
+        else:
+            add_parameter(control_parameter, control, dry_run, ssp_control)
+
+
+def update_parameter(ssp_parameter, control_parameter, control, dry_run):
+    ssp_parameter.parentParameterId = control_parameter.id
+    ssp_parameter.name = control_parameter.displayName
+    ssp_parameter.value = control_parameter.text
+    if ssp_parameter.has_changed():
+        _log_change(
+            "Update",
+            "Parameter",
+            f"Updating parameter {ssp_parameter.name} in control {control.controlId}",
+            changes=ssp_parameter.show_changes(),
+        )
+        if not dry_run:
+            ssp_parameter.save()
+
+
+def add_parameter(control_parameter, control, dry_run, ssp_control):
+    ssp_parameter = rm.Parameter(
+        uuid=uuid.uuid4().__str__(),
+        name=control_parameter.displayName,
+        value=control_parameter.text or "",
+        controlImplementationId=ssp_control.id,
+        parentParameterId=control_parameter.id,
+    )
+    _log_change(
+        "Add",
+        "Parameter",
+        f"Adding parameter {ssp_parameter.name} to control {control.controlId}",
+        changes=ssp_parameter.show_changes(),
+    )
+    if not dry_run:
+        ssp_parameter = ssp_parameter.create()
+
+
+def remove_unused_parameters(ssp_parameters, control_parameters, control, dry_run):
+    for parameter_id in ssp_parameters.keys():
+        if parameter_id not in control_parameters:
+            _log_change(
+                "Remove",
+                "Parameter",
+                f"Removing parameter {parameter_id} from control {control.controlId}",
+            )
+            if not dry_run:
+                ssp_parameters[parameter_id].delete()
+
+
+def get_controls_from_profile(ssp_id: int) -> Dict[str, rm.SecurityControl]:
+    """
+    Get controls from a profile
+
+    :param int ssp_id: The SSP ID
+    :return: The controls
+    :rtype: Dict[str, rm.SecurityControl]
+    """
+    logger.info(f"Getting controls from profile: {ssp_id}")
+    profile_links = rm.ProfileLink.get_all_by_parent(
+        parent_id=ssp_id, parent_module=rm.SecurityPlan.get_module_string()
+    )
+    controls: List[rm.SecurityControl] = []
+    control_get_count = 0
+    for profile_link in profile_links:
+        profile_mappings: list[rm.ProfileMapping] = rm.ProfileMapping.get_all_by_parent(
+            parent_id=profile_link.profileId, parent_module=rm.Profile.get_module_string()
+        )
+        for profile_mapping in profile_mappings:
+            control = rm.SecurityControl.get_object(object_id=profile_mapping.controlID)
+            if control:
+                print(control.controlId, end=" ")
+                controls.append(control)
+                control_get_count += 1
+                if control_get_count % 10 == 0:
+                    print()
+            else:
+                logger.warning(f"Control {profile_mapping.controlID} not found")
+    return {parentheses_to_dot(x.controlId): x for x in controls}
+
+
+def sync_plan_controls(ssp_id: int, dry_run: bool = False):
+    """
+    Sync controls from a profile to an SSP
+
+    :param int ssp_id: The SSP ID
+    :param bool dry_run: Dry run flag
+    """
+    logger.info(f"Syncing controls from profile to SSP: {ssp_id}")
+    log_entries.clear()
+    catalog_controls = get_controls_from_profile(ssp_id=ssp_id)
+    sync_controls(catalog_controls=catalog_controls, ssp_id=ssp_id, dry_run=dry_run)
+    _write_log_to_csv(f"plan_{ssp_id}_log_{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}.csv")
+    _log_summary()
+
+
+def sync_all_plans(dry_run: bool = False):
+    """
+    Sync all plans
+
+    :param bool dry_run: Dry run flag
+    """
+    for ssp in rm.SecurityPlan.get_list():  # type: rm.SecurityPlan
+        sync_plan_controls(ssp.id, dry_run)

regscale/core/app/utils/file_utils.py

@@ -0,0 +1,238 @@
+"""
+Utility functions for working with files and folders.
+"""
+
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Union, List, Iterator, Optional
+
+
+class S3FileDownloadError(Exception):
+    """
+    Exception raised when an error occurs during S3 file downloadß
+    """
+
+    pass
+
+
+def is_s3_path(path: Union[str, Path]) -> bool:
+    """
+    Check if the given path is an S3 URI.
+
+    :param Union[str, Path] path: The path to check
+    :return: True if the path is an S3 URI, False otherwise
+    :rtype: bool
+    """
+    return isinstance(path, str) and path.startswith("s3://")
+
+
+def read_file(file_path: Union[str, Path]) -> str:
+    """
+    Read a file from local filesystem or S3.
+
+    :param Union[str, Path] file_path: Path to the file or S3 URI
+    :return: Content of the file
+    :rtype: str
+    """
+    import smart_open  # type: ignore # Optimize import performance
+
+    with smart_open.open(str(file_path), "r") as f:
+        return f.read()
+
+
+def find_files(path: Union[str, Path], pattern: str) -> List[Union[Path, str]]:
+    """
+    Find all files matching the pattern in the given path, including S3.
+
+    :param Union[str, Path] path: Path to a file, a folder, or an S3 URI
+    :param str pattern: File pattern to match (e.g., "*.nessus")
+    :return: List of Path objects for matching files or S3 URIs
+    :rtype: List[Union[Path, str]]
+    """
+    import boto3  # type: ignore # Optimize import performance
+
+    if is_s3_path(path):
+        s3_parts = path[5:].split("/", 1)
+        bucket = s3_parts[0]
+        prefix = s3_parts[1] if len(s3_parts) > 1 else ""
+
+        s3 = boto3.client("s3")
+        paginator = s3.get_paginator("list_objects_v2")
+
+        files: List[Union[Path, str]] = []
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            for obj in page.get("Contents", []):
+                if obj["Key"].endswith(pattern.lstrip("*")):
+                    files.append(f"s3://{bucket}/{obj['Key']}")
+        return files
+
+    file_path = Path(path)
+    if file_path.is_file():
+        return [file_path] if file_path.match(pattern) else []
+    return list(file_path.glob(pattern))
+
+
+def move_file(src: Union[str, Path], dst: Union[str, Path]) -> None:
+    """
+    Move a file from src to dst. Works with local files and S3.
+
+    :param Union[str, Path] src: Source file path or S3 URI
+    :param Union[str, Path] dst: Destination file path or S3 URI
+    """
+    import smart_open  # type: ignore # Optimize import performance
+
+    if is_s3_path(src):
+        import boto3  # type: ignore # Optimize import performance
+
+        # S3 to S3 move
+        if is_s3_path(dst):
+            s3 = boto3.client("s3")
+            src_parts = src[5:].split("/", 1)
+            dst_parts = dst[5:].split("/", 1)
+            s3.copy_object(
+                CopySource={"Bucket": src_parts[0], "Key": src_parts[1]}, Bucket=dst_parts[0], Key=dst_parts[1]
+            )
+            s3.delete_object(Bucket=src_parts[0], Key=src_parts[1])
+        else:
+            # S3 to local
+            with smart_open.open(src, "rb") as s_file, smart_open.open(dst, "wb") as d_file:
+                d_file.write(s_file.read())
+            s3 = boto3.client("s3")
+            src_parts = src[5:].split("/", 1)
+            s3.delete_object(Bucket=src_parts[0], Key=src_parts[1])
+    else:
+        # Local to local or local to S3
+        with smart_open.open(src, "rb") as s_file, smart_open.open(dst, "wb") as d_file:
+            d_file.write(s_file.read())
+        if not isinstance(dst, str) or not dst.startswith("s3://"):
+            os.remove(src)
+
+
+def iterate_files(file_collection: List[Union[Path, str]]) -> Iterator[Union[Path, str]]:
+    """
+    Iterate over a collection of files, yielding each file path.
+
+    :param List[Union[Path, str]] file_collection: List of file paths or S3 URIs
+    :yield: Each file path or S3 URI
+    :rtype: Iterator[Union[Path, str]]
+    """
+    for file in file_collection:
+        yield file
+
+
+def get_processed_file_path(file_path: Union[str, Path], processed_folder: str = "processed") -> Union[str, Path]:
+    """
+    Generate a path for the processed file, handling both local and S3 paths.
+
+    :param Union[str, Path] file_path: Original file path or S3 URI
+    :param str processed_folder: Name of the folder for processed files (default: "processed")
+    :return: Path or S3 URI for the processed file
+    :rtype: Union[str, Path]
+    """
+    if is_s3_path(file_path):
+        s3_parts = file_path[5:].split("/")  # type: ignore  # is_s3_path ensures string
+        bucket = s3_parts[0]
+        key = "/".join(s3_parts[1:])
+        new_key = f"processed/{os.path.basename(key)}"
+        return f"s3://{bucket}/{new_key}"
+    else:
+        file_path = Path(file_path)
+        timestamp = datetime.now().strftime("%Y%m%d-%I%M%S%p")
+        new_filename = f"{file_path.stem}_{timestamp}{file_path.suffix}".replace(" ", "_")
+        new_path = file_path.parent / processed_folder / new_filename
+        os.makedirs(new_path.parent, exist_ok=True)
+        return new_path
+
+
+def get_files_by_folder(
+    import_folder: Union[str, Path],
+    exclude_non_scan_files: bool,
+    file_excludes: Optional[list[str]] = None,
+    directory_excludes: Optional[list[str]] = None,
+) -> List[str]:
+    """
+    Retrieves a list of file paths from a specified folder, excluding empty files or files that match the excludes list.
+
+    :param Union[str, Path] import_folder: The path to the folder from which to retrieve file paths.
+    :param bool exclude_non_scan_files: exclude files that are not scan files
+    :param Optional[List[str]] file_excludes: List of file extensions to exclude from the list of files
+    :param Optional[List[str]] directory_excludes: List of directories to exclude from the list of files
+    :return: A list of file paths for all non-empty files in the specified folder and subsequent subfolders.
+    :rtype: List[str]
+    """
+    if file_excludes is None:
+        file_excludes = []
+    if directory_excludes is None:
+        directory_excludes = []
+    file_path_list = []
+    if not os.path.isdir(import_folder):
+        from regscale.core.app.logz import create_logger
+
+        logger = create_logger()
+        logger.error(f"Folder '{import_folder}' does not exist.")
+        return file_path_list
+    for root, dir, files in os.walk(import_folder):
+        for file in files:
+            file_path = os.path.join(root, file)
+            if any(exclude_str in file_path for exclude_str in directory_excludes):
+                continue
+            if any(exclude_str in file for exclude_str in file_excludes) and exclude_non_scan_files:
+                continue
+            if os.path.getsize(file_path) == 0:
+                continue
+            file_path_list.append(os.path.join(root, file))
+    return file_path_list
+
+
+def download_from_s3(bucket: str, prefix: str, local_path: Union[str, os.PathLike], aws_profile: str) -> None:
+    """
+    Downloads files from an S3 bucket to a local directory.
+
+    :param str bucket: Name of the S3 bucket
+    :param str prefix: Prefix (folder path) within the bucket
+    :param Union[str, PathLike] local_path: Local directory to download files to
+    :param str aws_profile: AWS profile to use for S3 access
+    :rtype: None
+    """
+    import boto3
+    import logging
+    from botocore.exceptions import ClientError
+
+    logger = logging.getLogger(__name__)
+
+    session = boto3.Session(profile_name=aws_profile)
+    s3_client = session.client("s3")
+
+    try:
+        # Create local directory if it doesn't exist
+        os.makedirs(local_path, exist_ok=True)
+
+        # List objects in bucket with given prefix
+        paginator = s3_client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            if "Contents" not in page:
+                continue
+
+            for obj in page["Contents"]:
+                # Skip if object is a directory (ends with /)
+                if obj["Key"].endswith("/"):
+                    continue
+
+                # Create the full local path, preserving directory structure
+                relative_path = obj["Key"]
+                if prefix:
+                    # Remove the prefix from the key to get the relative path
+                    relative_path = relative_path[len(prefix) :].lstrip("/")
+
+                local_file_path = os.path.join(local_path, relative_path)
+
+                # Create the directory structure if it doesn't exist
+                os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+                logger.info(f"Downloading {obj['Key']} to {local_file_path}")
+                s3_client.download_file(bucket, obj["Key"], local_file_path)
+
+    except ClientError as e:
+        logger.error(f"Error downloading from S3: {str(e)}")
+        raise S3FileDownloadError(f"Failed to download files from S3: {str(e)}")

regscale/core/app/utils/parser_utils.py

@@ -0,0 +1,81 @@
+import datetime
+import logging
+from typing import Any, Optional
+
+from regscale.core.utils.date import datetime_str, date_str
+
+logger = logging.getLogger(__name__)
+
+
+def safe_float(value: Any, default: float = 0.0, field_name: str = "value") -> float:
+    """
+    Safely convert any value to a float.
+
+    :param Any value: The value to convert
+    :param float default: The default value to return if conversion fails
+    :param str field_name: The name of the field being parsed (for logging purposes)
+    :return: The parsed float value or the default value
+    :rtype: float
+    """
+    if value is None:
+        return default
+
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        logger.warning(f"Invalid float {field_name}: {value}. Defaulting to {default}")
+        return default
+
+
+def safe_int(value: Any, default: int = 0, field_name: str = "value") -> int:
+    """
+    Safely convert any value to an integer.
+
+    :param Any value: The value to convert
+    :param int default: The default value to return if conversion fails
+    :param str field_name: The name of the field being parsed (for logging purposes)
+    :return: The parsed integer value or the default value
+    :rtype: int
+    """
+    if value is None:
+        return default
+
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        logger.warning(f"Invalid integer {field_name}: {value}. Defaulting to {default}")
+        return default
+
+
+def safe_datetime_str(
+    value: Any, default: datetime.datetime = datetime.datetime.now(), date_format: Optional[str] = None
+) -> str:
+    """
+    Safely convert any value to a datetime.
+
+    :param Any value: The value to convert
+    :param datetime default: The default value to return if conversion fails
+    :param Optional[str] date_format: The date format to use for the datetime string, defaults to None
+    :return: The parsed datetime value or the default value
+    :rtype: str
+    """
+    value = datetime_str(value, date_format)
+    if not value:
+        value = datetime_str(default, date_format)
+    return value
+
+
+def safe_date_str(value: Any, default: datetime.date = datetime.date.today()) -> str:
+    """
+    Safely convert any value to a date string.
+
+    :param Any value: The value to convert
+    :param date default: The default value to return if conversion fails
+    :return: The parsed date string value or the default value
+
+    :rtype: str
+    """
+    value = date_str(value)
+    if not value:
+        value = date_str(default)
+    return value