regscale-cli 6.25.1.0__py3-none-any.whl → 6.27.0.0__py3-none-any.whl

regscale/_version.py

@@ -33,7 +33,7 @@ def get_version_from_pyproject() -> str:
                     return match.group(1)
     except Exception:
         pass
-    return "6.25.1.0"  # fallback version
+    return "6.27.0.0"  # fallback version
 
 
 __version__ = get_version_from_pyproject()

regscale/airflow/hierarchy.py

@@ -2,9 +2,9 @@
 
 from typing import Optional
 
-from regscale.regscale import cli
-from regscale.models.click_models import ClickCommand
 from regscale.airflow.click_mixins import AirflowClickGroup
+from regscale.models.click_models import ClickCommand
+from regscale.regscale import cli
 
 AIRFLOW_CLICK_GROUP = AirflowClickGroup.from_group(cli, prefix="regscale")
 AIRFLOW_CLICK_OPERATORS = AIRFLOW_CLICK_GROUP.flatten_operator()

regscale/core/app/application.py

@@ -269,7 +269,7 @@ class Application(metaclass=Singleton):
             "token": DEFAULT_POPULATED,
             "userId": "enter RegScale user id here",
             "useMilestones": False,
-            "preventAutoClose": True,
+            "preventAutoClose": False,
             "otx": "enter AlienVault API key here",
             "wizAccessToken": DEFAULT_POPULATED,
             "wizAuthUrl": "https://auth.wiz.io/oauth/token",
@@ -664,7 +664,8 @@ class Application(metaclass=Singleton):
 
     def save_config(self, conf: dict) -> None:
         """
-        Save Configuration to init.yaml
+        Save Configuration to init.yaml using atomic file operations to prevent corruption
+        during parallel writes.
 
         :param dict conf: Application configuration
         :rtype: None
@@ -681,8 +682,22 @@ class Application(metaclass=Singleton):
         try:
             self.logger.debug(f"Saving config to {self.config_file}.")
             with self._config_lock:
-                with open(self.config_file, "w", encoding="utf-8") as file:
-                    yaml.dump(conf, file)
+                # Use atomic file operations: write to temp file, then rename
+                # This prevents corruption when multiple processes write simultaneously
+                import tempfile
+
+                config_dir = os.path.dirname(self.config_file) or "."
+                temp_fd, temp_path = tempfile.mkstemp(dir=config_dir, prefix=".tmp_", suffix=".yaml", text=True)
+                try:
+                    with os.fdopen(temp_fd, "w", encoding="utf-8") as temp_file:
+                        yaml.dump(conf, temp_file)
+                    # Atomic rename - this is atomic on POSIX systems
+                    os.replace(temp_path, self.config_file)
+                except Exception:
+                    # Clean up temp file if something goes wrong
+                    with contextlib.suppress(OSError):
+                        os.unlink(temp_path)
+                    raise
         except OSError:
             self.logger.error(f"Could not save config to {self.config_file}.")
 

regscale/core/app/internal/evidence.py

@@ -24,7 +24,8 @@ from regscale.core.app.api import Api
 from regscale.core.app.application import Application
 from regscale.core.app.utils.app_utils import check_file_path, create_progress_object, error_and_exit
 from regscale.models.app_models.click import regscale_ssp_id
-from regscale.models.regscale_models import Assessment, File, Project, SecurityPlan
+from regscale.models.regscale_models import Assessment, File, Project, SecurityPlan, Evidence, Component
+from regscale.models.regscale_models.control_implementation import ControlImplementation
 
 logger = getLogger("regscale")
 
@@ -198,7 +199,7 @@ def package_builder(ssp_id: int, path: Path):
     app = Application()
     api = Api()
     with create_progress_object() as progress:
-        task = progress.add_task("[white]Building and zipping evidence folder for audit...", total=6)
+        task = progress.add_task("[white]Building and zipping evidence folder for audit...", total=8)
         try:
             # Obtaining MEGA Api for given Organizer Record.
             ssp = SecurityPlan.fetch_mega_api_data(ssp_id)
@@ -221,6 +222,16 @@ def package_builder(ssp_id: int, path: Path):
 
             progress.update(task, advance=1)
 
+            # Process evidence lockers at SSP level
+            process_ssp_evidence_lockers(
+                ssp_id=ssp_id,
+                path=path,
+                module_folder=module_folder,
+                api=api,
+            )
+
+            progress.update(task, advance=1)
+
             # Checking MEGA Api for Attachments at Control level
             process_control_attachments(
                 ssp=ssp,
@@ -231,6 +242,19 @@ def package_builder(ssp_id: int, path: Path):
                 api=api,
                 task=task,
             )
+
+            progress.update(task, advance=1)
+
+            # Process components and their evidence
+            process_components_evidence(
+                ssp_id=ssp_id,
+                path=path,
+                module_folder=module_folder,
+                api=api,
+            )
+
+            progress.update(task, advance=1)
+
             # Creating zip file and removing temporary Evidence Folder
             new_path = Path("./evidence.zip")
             zip_folder(path, new_path)
@@ -349,6 +373,9 @@ def process_control_attachments(
         # Adding any Attachments at Control level to corresponding folder
         _download_control_attachments(control_attachments, api, path, module_folder_name)
 
+        # Process evidence lockers for controls
+        _process_control_evidence_lockers(control_attachments, api, path, module_folder_name)
+
         progress.update(task, advance=1)
 
     else:
@@ -388,6 +415,396 @@ def _download_control_attachments(
             json.dump(f, file_drop, indent=4, separators=(", ", ": "))
 
 
+def _get_control_folder_name(control_attachments: list[dict], control_id: int) -> str | None:
+    """
+    Get the control folder name for a given control ID
+
+    :param list[dict] control_attachments: List of control attachments
+    :param int control_id: Control ID to find folder name for
+    :return: Control folder name or None
+    :rtype: str | None
+    """
+    for f in control_attachments:
+        if f["parentId"] == control_id:
+            return f["controlId"]
+    return None
+
+
+def _download_control_evidence_items(
+    evidence_items: list[dict], control_folder_name: str, path: Path, module_folder_name: str, api: Api
+) -> None:
+    """
+    Download evidence items for a control
+
+    :param list[dict] evidence_items: List of evidence items
+    :param str control_folder_name: Name of the control folder
+    :param Path path: Base path for downloads
+    :param str module_folder_name: Module folder name
+    :param Api api: API object
+    :rtype: None
+    """
+    logger.info(f"Found {len(evidence_items)} evidence items for control {control_folder_name}")
+
+    for evidence_item in evidence_items:
+        file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+        output_path = f"{path}/{module_folder_name}/{control_folder_name}/{file_name}"
+
+        if download_evidence_file(api, evidence_item, output_path):
+            logger.info(f"Downloaded evidence file: {file_name}")
+        else:
+            logger.warning(f"Failed to download evidence file: {file_name}")
+
+
+def _process_control_evidence_lockers(
+    control_attachments: list[dict], api: Api, path: Path, module_folder_name: str
+) -> None:
+    """
+    Process evidence lockers for controls
+
+    :param list[dict] control_attachments: List of control attachments
+    :param Api api: RegScale CLI API object
+    :param Path path: directory for file location
+    :param str module_folder_name: name of the module folder
+    :rtype: None
+    """
+    # Get unique control IDs
+    control_ids = list({f["parentId"] for f in control_attachments})
+
+    for control_id in control_ids:
+        try:
+            # Get evidence from evidence lockers for this control
+            evidence_items = get_evidence_by_control(api, control_id)
+
+            if evidence_items:
+                # Find the control ID for folder naming
+                control_folder_name = _get_control_folder_name(control_attachments, control_id)
+
+                if control_folder_name:
+                    _download_control_evidence_items(evidence_items, control_folder_name, path, module_folder_name, api)
+        except Exception as e:
+            logger.warning(f"Failed to process evidence lockers for control {control_id}: {e}")
+
+
+def get_evidence_by_control(api: Api, control_id: int) -> list[dict]:
+    """
+    Get evidence for a specific control
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int control_id: Control ID
+    :return: List of evidence items
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Evidence model method instead of direct API call
+        evidence_items = Evidence.get_all_by_parent(parent_id=control_id, parent_module="controls")
+        # Convert to dict format for compatibility
+        return [evidence.dict() for evidence in evidence_items]
+    except Exception as e:
+        logger.warning(f"Failed to get evidence for control {control_id}: {e}")
+        return []
+
+
+def get_evidence_by_security_plan(api: Api, ssp_id: int) -> list[dict]:
+    """
+    Get evidence for a specific security plan
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int ssp_id: Security Plan ID
+    :return: List of evidence items
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Evidence model method instead of direct API call
+        evidence_items = Evidence.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
+        # Convert to dict format for compatibility
+        return [evidence.dict() for evidence in evidence_items]
+    except Exception as e:
+        logger.warning(f"Failed to get evidence for security plan {ssp_id}: {e}")
+        return []
+
+
+def get_components_by_ssp(api: Api, ssp_id: int) -> list[dict]:
+    """
+    Get components for a specific security plan
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int ssp_id: Security Plan ID
+    :return: List of active components
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Component model method instead of direct API call
+        components = Component.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
+        # Filter for active components only and convert to dict format
+        return [comp.dict() for comp in components if comp.status == "Active"]
+    except Exception as e:
+        logger.warning(f"Failed to get components for security plan {ssp_id}: {e}")
+        return []
+
+
+def get_controls_by_parent(api: Api, parent_id: int, parent_module: str) -> list[dict]:
+    """
+    Get controls for a specific parent (SSP or Component)
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int parent_id: Parent ID
+    :param str parent_module: Parent module (securityplans or components)
+    :return: List of controls
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use ControlImplementation model method instead of direct API call
+        controls = ControlImplementation.get_all_by_parent(parent_id=parent_id, parent_module=parent_module)
+        # Convert to dict format for compatibility
+        return [control.dict() for control in controls]
+    except Exception as e:
+        logger.warning(f"Failed to get controls for parent {parent_id} in module {parent_module}: {e}")
+        return []
+
+
+def download_evidence_file(api: Api, evidence_item: dict, output_path: str) -> bool:
+    """
+    Download an evidence file
+
+    :param Api api: RegScale CLI API object
+    :param dict evidence_item: Evidence item data
+    :param str output_path: Path to save the file
+    :return: True if successful, False otherwise
+    :rtype: bool
+    """
+    try:
+        file_data = File.download_file_from_regscale_to_memory(
+            api=api,
+            record_id=evidence_item["parentId"],
+            module=evidence_item["parentModule"],
+            stored_name=evidence_item["trustedStorageName"],
+            file_hash=evidence_item.get("fileHash") or evidence_item.get("shaHash"),
+        )
+
+        if file_data is None:
+            logger.warning(f"No data received for evidence file {evidence_item.get('trustedDisplayName', 'unknown')}")
+            return False
+
+        with open(output_path, "wb") as f:
+            f.write(file_data)
+        return True
+    except Exception as e:
+        logger.warning(f"Failed to download evidence file {evidence_item.get('trustedDisplayName', 'unknown')}: {e}")
+        return False
+
+
+def process_ssp_evidence_lockers(ssp_id: int, path: Path, module_folder: Path, api: Api) -> None:
+    """
+    Process evidence lockers at SSP level
+
+    :param int ssp_id: Security Plan ID
+    :param Path path: directory for file location
+    :param str module_folder_name: name of the module folder
+    :param Path module_folder: path to module folder
+    :param Api api: RegScale CLI API object
+    :rtype: None
+    """
+    try:
+        # Get evidence from evidence lockers for the SSP
+        evidence_items = get_evidence_by_security_plan(api, ssp_id)
+
+        if evidence_items:
+            logger.info(f"Found {len(evidence_items)} evidence items from evidence lockers for SSP {ssp_id}")
+
+            for evidence_item in evidence_items:
+                file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+                output_path = module_folder / file_name
+
+                if download_evidence_file(api, evidence_item, str(output_path)):
+                    logger.info(f"Downloaded evidence file: {file_name}")
+                else:
+                    logger.warning(f"Failed to download evidence file: {file_name}")
+        else:
+            logger.info("No evidence found in evidence lockers for SSP")
+
+    except Exception as e:
+        logger.warning(f"Error processing SSP evidence lockers: {e}")
+
+
+def _download_files_for_parent(
+    parent_id: int, parent_module: str, output_folder: Path, api: Api, module_name: str = None
+) -> None:
+    """
+    Generalized function to download files for any parent module
+
+    :param int parent_id: Parent ID (component, control, etc.)
+    :param str parent_module: Parent module name (components, controls, etc.)
+    :param Path output_folder: Path to output folder
+    :param Api api: API object
+    :param str module_name: Human-readable module name for logging (optional)
+    :rtype: None
+    """
+    if module_name is None:
+        module_name = parent_module
+
+    try:
+        # Use File model method instead of direct API call
+        files_data = File.get_files_for_parent_from_regscale(api=api, parent_id=parent_id, parent_module=parent_module)
+
+        for file_item in files_data:
+            file_name = file_item.trustedDisplayName or f"file_{file_item.id}"
+            output_path = output_folder / file_name
+
+            try:
+                file_data = File.download_file_from_regscale_to_memory(
+                    api=api,
+                    record_id=file_item.id,
+                    module=parent_module,
+                    stored_name=file_item.trustedStorageName,
+                    file_hash=file_item.fileHash or file_item.shaHash,
+                )
+
+                if file_data is None:
+                    logger.warning(f"No data received for {module_name} file {file_name}")
+                    continue
+
+                with open(output_path, "wb") as f:
+                    f.write(file_data)
+                logger.info(f"Downloaded {module_name} file: {file_name}")
+            except Exception as e:
+                logger.warning(f"Failed to download {module_name} file {file_name}: {e}")
+    except Exception as e:
+        logger.warning(f"Failed to get {module_name} files for {parent_module} {parent_id}: {e}")
+
+
+def _download_component_files(component_id: int, component_folder: Path, api: Api) -> None:
+    """
+    Download files directly attached to a component
+
+    :param int component_id: Component ID
+    :param Path component_folder: Path to component folder
+    :param Api api: API object
+    :rtype: None
+    """
+    _download_files_for_parent(
+        parent_id=component_id,
+        parent_module="components",
+        output_folder=component_folder,
+        api=api,
+        module_name="component",
+    )
+
+
+def _download_control_files(control_id: int, control_folder: Path, api: Api) -> None:
+    """
+    Download files for a control
+
+    :param int control_id: Control ID
+    :param Path control_folder: Path to control folder
+    :param Api api: API object
+    :rtype: None
+    """
+    _download_files_for_parent(
+        parent_id=control_id, parent_module="controls", output_folder=control_folder, api=api, module_name="control"
+    )
+
+
+def _download_control_evidence(control_id: int, control_folder: Path, api: Api) -> None:
+    """
+    Download evidence from evidence lockers for a control
+
+    :param int control_id: Control ID
+    :param Path control_folder: Path to control folder
+    :param Api api: API object
+    :rtype: None
+    """
+    evidence_items = get_evidence_by_control(api, control_id)
+
+    if evidence_items:
+        logger.info(f"Found {len(evidence_items)} evidence items for control {control_folder.name}")
+
+        for evidence_item in evidence_items:
+            file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+            output_path = control_folder / file_name
+
+            if download_evidence_file(api, evidence_item, str(output_path)):
+                logger.info(f"Downloaded evidence file: {file_name}")
+            else:
+                logger.warning(f"Failed to download evidence file: {file_name}")
+
+
+def _process_component_controls(component_id: int, component_folder: Path, api: Api) -> None:
+    """
+    Process controls for a component
+
+    :param int component_id: Component ID
+    :param Path component_folder: Path to component folder
+    :param Api api: API object
+    :rtype: None
+    """
+    controls = get_controls_by_parent(api, component_id, "components")
+
+    if controls:
+        logger.info(f"Found {len(controls)} controls for component {component_folder.name}")
+
+        for control in controls:
+            control_id = control.get("id")
+            control_name = control.get("controlId", f"Control_{control_id}")
+
+            # Create control folder within component folder
+            control_folder = component_folder / control_name
+            os.makedirs(control_folder, exist_ok=True)
+
+            # Download control files and evidence
+            _download_control_files(control_id, control_folder, api)
+            _download_control_evidence(control_id, control_folder, api)
+
+
+def process_components_evidence(ssp_id: int, path: Path, module_folder: Path, api: Api) -> None:
+    """
+    Process components and their evidence
+
+    :param int ssp_id: Security Plan ID
+    :param Path path: directory for file location
+    :param Path module_folder: path to module folder
+    :param Api api: RegScale CLI API object
+    :rtype: None
+    """
+    try:
+        # Get components for the SSP
+        components = get_components_by_ssp(api, ssp_id)
+
+        if not components:
+            logger.info("No active components found for SSP")
+            return
+
+        logger.info(f"Found {len(components)} active components for SSP {ssp_id}")
+
+        for component in components:
+            component_id = component.get("id")
+            component_title = component.get("title", f"Component_{component_id}")
+
+            # Create component folder
+            component_folder = module_folder / component_title
+            os.makedirs(component_folder, exist_ok=True)
+
+            # Download component files
+            _download_component_files(component_id, component_folder, api)
+
+            # Process component controls
+            _process_component_controls(component_id, component_folder, api)
+
+    except Exception as e:
+        logger.warning(f"Error processing components evidence: {e}")
+
+
 def remove_directory(directory_path: Path) -> None:
     """
     This function removes a given directory even if files stored there

regscale/core/app/internal/login.py

@@ -136,7 +136,6 @@ def login(
         logger.info("New RegScale Token has been updated and saved in init.yaml")
         # Truncate token for logging purposes
         logger.debug("Token: %s", regscale_auth.token[:20])
-    config["domain"] = host
     app.save_config(config)
     return regscale_auth.token
 

regscale/core/app/utils/catalog_utils/common.py

@@ -87,5 +87,5 @@ def objective_to_control_dot(input_string: str) -> str:
     if match:
         return match.group(1)
     else:
-        logger.error(f"Failed to convert objective to control: {input_string}")
+        logger.debug(f"Failed to convert objective to control: {input_string}")
         return input_string

regscale/dev/code_gen.py

@@ -170,29 +170,27 @@ def _build_op_kwargs_and_docstring(
         capabilities=capabilities,
     )
 
-    if ConnectorType.Vulnerabilities.lower() in integration and param_type == "optional_params":
-        if param_type == "optional_params" and ConnectorType.Vulnerabilities in integration:
+    if ConnectorType.Vulnerabilities.lower() in integration:
+        vuln_params = {}
+        if param_type == "optional_params":
             vuln_params: dict[str, Param] = {
-                "vuln_filter": Param(
-                    name="vuln_filter",
-                    type="choice",
-                    description="Filter the vulnerabilities for the selected severity. (Options: critical, high, medium, low, info)",
-                    default=None,
-                ),
                 "scan_date": Param(
                     name="scan_date",
                     type="string",
                     description="The date of the scan to sync vulnerabilities into RegScale.",
                     default=None,
                 ),
-                "all_scans": Param(
-                    name="all_scans",
-                    type="boolean",
-                    description="Whether to sync all vulnerabilities into RegScale.",
-                    default=False,
+            }
+        elif param_type == "expected_params":
+            vuln_params: dict[str, Param] = {
+                "minimum_severity_filter": Param(
+                    name="minimum_severity_filter",
+                    type="choice",
+                    description="Minimum severity of the vulnerabilities to sync. (Options: critical, high, medium, low, info, all), e.g. providing 'high' will sync all vulnerabilities with a severity of high and critical.",
+                    default=None,
                 ),
             }
-            config[param_type] = {**config[param_type], **vuln_params}
+        config[param_type] = {**config[param_type], **vuln_params}
 
     # Add filter parameter for Assets and Vulnerabilities connectors
     if (
@@ -214,7 +212,7 @@ def _build_op_kwargs_and_docstring(
             doc_string += f"{proper_type}:\n"
         for param in config[param_type]:
             op_kwargs, doc_string = _build_other_params(
-                config=config,
+                param_obj=config[param_type][param],
                 param=param,
                 param_type=param_type,
                 proper_type=proper_type,
@@ -258,11 +256,13 @@ def _build_expected_params(
 
 
 def _build_other_params(
-    config: dict, param: str, param_type: str, proper_type: str, integration: str, op_kwargs: str, doc_string: str
+    param_obj: Param, param: str, param_type: str, proper_type: str, integration: str, op_kwargs: str, doc_string: str
 ) -> tuple[str, str]:
     """
     Build the other params for the DAG by adding them to the op_kwargs and docstring
 
+    :param Param param_obj: The parameter object
+    :param str param: The name of the parameter
     :param str param_type: The type of parameter to build
     :param str integration: The name of the integration, typically connector_integration
     :param str op_kwargs: The op_kwargs to add to the DAG
@@ -270,7 +270,6 @@ def _build_other_params(
     :return: The op_kwargs and docstring
     :rtype: tuple[str, str]
     """
-    param_obj: Param = config[param_type][param]
     param_name = f"{integration.lower()}_{param}" if param_type == "required_secrets" else param
     jinja_key = f"'{param_name}'"
     if default := param_obj.default:
@@ -447,14 +446,19 @@ def _build_all_params(
     :rtype: tuple[list[str], list[str], list[str]]
     """
     if connector == ConnectorType.Vulnerabilities:
-        vuln_filter_option = "@click.option(\n    '--vuln_filter',\n    help='Filter the vulnerabilities for the selected severity. (Options: critical, high, medium, low, info)',\n    required=False,\n    type=click.Choice(['critical', 'high', 'medium', 'low', 'info']),\n    default=None)\n"
+        vuln_filter_option = "@click.option(\n    '--minimum_severity_filter','-s',\n    help='Minimum severity of the vulnerabilities to sync. (Options: critical, high, medium, low, info), e.g. providing high will sync all vulnerabilities with a severity of high and critical.',\n    required=False,\n    type=click.Choice(['critical', 'high', 'medium', 'low', 'info']),\n    default=None)\n"
         scan_date_option = f"@click.option(\n    '--scan_date',\n    help='The date of the scan to sync vulnerabilities from {integration_name}',\n    required=False,\n    type=click.DateTime(formats=['%Y-%m-%d']),\n    default=None)\n"
         all_vulns_flag = f"@click.option(\n    '--all_scans',\n    help='Whether to sync all vulnerabilities from {integration_name}',\n    required=False,\n    is_flag=True,\n    default=False)\n"
         click_options = ["@regscale_ssp_id()", vuln_filter_option, scan_date_option, all_vulns_flag]
-        function_params = ["regscale_ssp_id: int", "vuln_filter: str", "scan_date: datetime", "all_scans: bool"]
+        function_params = [
+            "regscale_ssp_id: int",
+            "minimum_severity_filter: str",
+            "scan_date: datetime",
+            "all_scans: bool",
+        ]
         function_kwargs = [
             "regscale_ssp_id=regscale_ssp_id",
-            "vuln_filter=vuln_filter",
+            "minimum_severity_filter=minimum_severity_filter",
             "scan_date=scan_date",
             "all_scans=all_scans",
         ]