regscale-cli 6.16.3.0__py3-none-any.whl → 6.16.4.0__py3-none-any.whl

regscale/integrations/public/fedramp/fedramp_cis_crm.py

@@ -2,23 +2,27 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=C0415
 """standard python imports"""
+from __future__ import annotations
+
 import json
 import math
 import re
 from collections import Counter
 from concurrent.futures import as_completed
 from concurrent.futures.thread import ThreadPoolExecutor
-from typing import Dict, List, Literal, Optional, Tuple
-from urllib.parse import urljoin
+from threading import Thread
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, TypeVar
 
 import click
 
 from regscale.core.app.api import Api
+from regscale.core.app.utils.api_handler import APIInsertionError, APIUpdateError
 from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit, get_current_datetime
 from regscale.core.utils.graphql import GraphQLQuery
 from regscale.integrations.public.fedramp.parts_mapper import PartMapper
 from regscale.integrations.public.fedramp.ssp_logger import SSPLogger
-from regscale.models import ControlObjective, ImplementationObjective
+from regscale.models import ControlObjective, ImplementationObjective, Parameter, Profile
 from regscale.models.regscale_models import (
     ControlImplementation,
     File,
@@ -26,7 +30,17 @@ from regscale.models.regscale_models import (
     SecurityControl,
     SecurityPlan,
 )
+from regscale.models.regscale_models.compliance_settings import ComplianceSettings
 from regscale.models.regscale_models.control_implementation import ControlImplementationStatus
+from regscale.utils.threading import ThreadSafeDict, ThreadSafeSet
+
+# For type annotations only
+if TYPE_CHECKING:
+    import pandas as pd
+
+from functools import lru_cache
+
+T = TypeVar("T")
 
 logger = SSPLogger()
 part_mapper_rev5 = PartMapper()
@@ -49,6 +63,9 @@ IMPACT_LEVEL = "Impact Level"
 SYSTEM_NAME = "System Name"
 CSP = "CSP"
 
+EXISTING_IMPLEMENTATIONS: ThreadSafeDict[int, ControlImplementation] = ThreadSafeDict()
+UPDATED_IMPLEMENTATION_OBJECTIVES: ThreadSafeSet[ImplementationObjective] = ThreadSafeSet()
+
 STATUS_MAPPING = {
     "Implemented": ControlImplementationStatus.Implemented,
     PARTIALLY_IMPLEMENTED: ControlImplementationStatus.PartiallyImplemented,
@@ -76,6 +93,20 @@ RESPONSIBILITY_MAP = {
     "bShared": "Shared",
     "bInherited": "Inherited",
 }
+REGSCALE_SSP_ID: int = 0
+
+
+@lru_cache(maxsize=1)
+def get_pandas() -> ModuleType:
+    """
+    Lazily import pandas only once when needed
+
+    :return: The pandas module
+    :rtype: ModuleType
+    """
+    import pandas as pd
+
+    return pd
 
 
 def transform_control(control: str) -> str:
@@ -280,17 +311,17 @@ def update_imp_objective(
     imp: ControlImplementation,
     objectives: List[ControlObjective],
     record: dict,
-) -> Optional[ImplementationObjective]:
+) -> None:
     """
-    Update the control objective with the given record data.
+    Update the control objectives with the given record data.
 
     :param int leverage_auth_id: The leveraged authorization ID
     :param List[ImplementationObjective] existing_imp_obj: The existing implementation objective
     :param ControlImplementation imp: The control implementation to update
     :param List[ControlObjective] objectives: The control objective to update
     :param dict record: The CIS/CRM record data to update the objective with
-    :rtype: Optional[ImplementationObjective]
-    :return: The updated or created implementation objective
+    :rtype: None
+    :return: None
     """
     status_map = {
         "Implemented": ControlImplementationStatus.Implemented.value,
@@ -312,32 +343,49 @@ def update_imp_objective(
 
     cis_record = record.get("cis", {})
     crm_record = record.get("crm", {})
-    responsibility = RESPONSIBILITY_MAP.get(
-        cis_record.get("control_origination", ""), ControlImplementationStatus.NA.value
-    )
+    # There could be multiples, take the first one as regscale will not allow multiples at the objective level.
+    control_originations = cis_record.get("control_origination", "").split(",")
+    for ix, control_origination in enumerate(control_originations):
+        control_originations[ix] = control_origination.strip()
+
+    try:
+        responsibility = RESPONSIBILITY_MAP.get(
+            next(origin for origin in control_originations), SERVICE_PROVIDER_CORPORATE
+        )
+    except StopIteration:
+        responsibility = SERVICE_PROVIDER_CORPORATE
+
     customer_responsibility = clean_customer_responsibility(
         crm_record.get("specific_inheritance_and_customer_agency_csp_responsibilities")
     )
-    ret_objective = None
     existing_pairs = {(obj.objectiveId, obj.implementationId) for obj in existing_imp_obj}
+    responsibility = responsibility_map.get(responsibility, responsibility)
     for objective in objectives:
         current_pair = (objective.id, imp.id)
         if current_pair not in existing_pairs:
             imp_obj = ImplementationObjective(
                 id=0,
                 uuid="",
-                inherited=crm_record.get("can_be_inherited_from_csp") == "Yes",
+                inherited=crm_record.get("can_be_inherited_from_csp") in ["Yes", "Partial"],
                 implementationId=imp.id,
                 status=status_map.get(cis_record.get("implementation_status", NOT_IMPLEMENTED), NOT_IMPLEMENTED),
                 objectiveId=objective.id,
                 notes=objective.name,
                 securityControlId=objective.securityControlId,
-                responsibility=responsibility_map.get(responsibility, responsibility),
+                securityPlanId=REGSCALE_SSP_ID,
+                responsibility=responsibility,
                 cloudResponsibility=customer_responsibility,
                 customerResponsibility=customer_responsibility,
                 authorizationId=leverage_auth_id,
+                parentObjectiveId=objective.parentObjectiveId,
+            )
+            logger.debug(
+                "Creating new Implementation Objective for Control %s with status: %s responsibility: %s",
+                imp_obj.securityControlId,
+                imp_obj.status,
+                imp_obj.responsibility,
             )
-            ret_objective = imp_obj.create()
+            UPDATED_IMPLEMENTATION_OBJECTIVES.add(imp_obj)
         else:
             # NOTE: Don't overwrite the responsibility text and only append.
             ex_obj = next((obj for obj in existing_imp_obj if obj.objectiveId == objective.id), None)
@@ -355,21 +403,19 @@ def update_imp_objective(
                         )
                     if ex_obj.cloudResponsibility:
                         ex_obj.cloudResponsibility = (
-                            seperator.join([ex_obj.cloudResponsibility, responsibility])
+                            seperator.join([ex_obj.cloudResponsibility, customer_responsibility])
                             if ex_obj.cloudResponsibility != responsibility
                             else ex_obj.cloudResponsibility
                         )
                     if ex_obj.customerResponsibility:
                         ex_obj.customerResponsibility = (
-                            seperator.join([ex_obj.customerResponsibility, responsibility])
+                            seperator.join([ex_obj.customerResponsibility, customer_responsibility])
                             if ex_obj.cloudResponsibility != responsibility
                             else ex_obj.customerResponsibility
                         )
                 except TypeError:
                     logger.warning(f"Failed to update responsibility on Implementation Objective #{ex_obj.id}")
-                ret_objective = ex_obj.save()
-
-    return ret_objective
+                UPDATED_IMPLEMENTATION_OBJECTIVES.add(ex_obj)
 
 
 def parse_control_details(
@@ -456,62 +502,62 @@ def fetch_and_update_imps(
     :rtype: Optional[ControlImplementation]
     """
     # get the control and control implementation objects
-    regscale_control = SecurityControl.get_object(control["scId"])
-    regscale_control_imp = ControlImplementation.get_object(control["id"])
+    regscale_control = SecurityControl.get_object(control.controlID)
+    if not regscale_control:
+        api.logger.error(f"Failed to fetch control with ID {control['scId']}")
+        return None
 
-    if not regscale_control or not regscale_control_imp:
-        api.logger.error("Failed to fetch control or control implementation")
-        return regscale_control_imp
+    control_id = regscale_control.controlId if regscale_control else ""
+    security_control_id = regscale_control.id if regscale_control else 0
+    regscale_control_imp = EXISTING_IMPLEMENTATIONS.get(security_control_id)
+
+    if not regscale_control_imp:
+        api.logger.error(f"Failed to find control implementation for control ID {control_id}")
+        return None
 
     updated_control = parse_control_details(
         version=version, control_imp=regscale_control_imp, control=regscale_control, cis_data=cis_data
     )
+
+    # Find the index of the old implementation and replace it with the updated one
+    EXISTING_IMPLEMENTATIONS[updated_control.controlID] = updated_control
+
     return updated_control
 
 
-def get_all_imps(api: Api, ssp_id: int, cis_data: dict, version: Literal["rev4", "rev5"]) -> list:
+def get_all_imps(api: Api, cis_data: dict, version: Literal["rev4", "rev5"]) -> None:
     """
     Function to retrieve control implementations and their objectives from RegScale
 
     :param Api api: The RegScale API object
-    :param int ssp_id: The SSP ID
     :param dict cis_data: The data from the CIS worksheet
     :param Literal["rev4", "rev5"] version: The version of the workbook
-    :return: List of updated control implementations
-    :rtype: list
+    :return: None
+    :rtype: None
     """
     from requests import RequestException
 
-    updated_controls = []
-    url = urljoin(api.config["domain"], f"/api/controlImplementation/getSCListByPlan/{ssp_id}")
-    response = api.get(url)
-
-    if response.status_code == 404:
-        api.logger.warning(f"SSP with ID {ssp_id} has no controls.")
-        return updated_controls
-
     # Check if the response is successful
-    if response.status_code == 200:
-        ssp_controls = response.json()
+    if EXISTING_IMPLEMENTATIONS:
         # Get Control Implementations For SSP
         fetching_imps = progress.add_task(
-            f"[magenta]Fetching & updating {len(ssp_controls)} implementation(s)...", total=len(ssp_controls)
+            f"[magenta]Updating {len(EXISTING_IMPLEMENTATIONS)} implementation(s)...",
+            total=len(EXISTING_IMPLEMENTATIONS),
         )
         with ThreadPoolExecutor(max_workers=50) as executor:
             futures = [
-                executor.submit(fetch_and_update_imps, control, api, cis_data, version) for control in ssp_controls
+                executor.submit(fetch_and_update_imps, control, api, cis_data, version)
+                for control in EXISTING_IMPLEMENTATIONS.values()
             ]
+
+            # Just wait for all tasks to complete
             for future in as_completed(futures):
-                progress.update(fetching_imps, advance=1)
                 try:
-                    controls = future.result()
-                    updated_controls.append(controls)
-                except (RequestException, TimeoutError) as ex:
-                    api.logger.error(f"Error fetching control implementations: {ex}")
-    else:
-        api.logger.error(f"Failed to fetch controls: {response.status_code}: {response.reason}")
-
-    return updated_controls
+                    # Only call result() to propagate any exceptions
+                    future.result()
+                    progress.update(fetching_imps, advance=1)
+                except Exception as e:
+                    logger.error(f"Error updating implementation: {e}")
 
 
 def get_all_control_objectives(imps: List[ControlImplementation]) -> List[ControlObjective]:
@@ -543,7 +589,6 @@ def update_all_objectives(
     leveraged_auth_id: int,
     cis_data: Dict[str, Dict[str, str]],
     crm_data: Dict[str, Dict[str, str]],
-    control_implementations: List[ControlImplementation],
     version: Literal["rev4", "rev5"],
 ) -> set:
     """
@@ -553,25 +598,27 @@ def update_all_objectives(
     :param int leveraged_auth_id: The leveraged authorization ID
     :param Dict[str, Dict[str, str]] cis_data: The CIS data to update from
     :param Dict[str, Dict[str, str]] crm_data: The CRM data to update from
-    :param List[ControlImplementation] control_implementations: The control implementations to update
     :param Literal["rev4", "rev5"] version: The version of the workbook
     :return: A set of errors, if any
     :rtype: set
     """
-    all_control_objectives = get_all_control_objectives(imps=control_implementations)
+
+    all_control_objectives = get_all_control_objectives(imps=EXISTING_IMPLEMENTATIONS.values())
     error_set = set()
-    task = progress.add_task("[cyan]Processing control objectives...", total=len(control_implementations))
+    process_task = progress.add_task(
+        "[cyan]Processing control objectives...", total=len(EXISTING_IMPLEMENTATIONS.values())
+    )
     # Create a combined dataset for easier access
     combined_data = {key: {"cis": cis_data[key], "crm": crm_data.get(key, {})} for key in cis_data}
 
     # Process implementations in parallel
-    with ThreadPoolExecutor(max_workers=50) as executor:
+    with ThreadPoolExecutor(max_workers=30) as executor:
         # Submit all tasks
         future_to_control = {
             executor.submit(
                 process_implementation, leveraged_auth_id, imp, combined_data, version, all_control_objectives
             ): imp
-            for imp in control_implementations
+            for imp in EXISTING_IMPLEMENTATIONS.values()
         }
 
         # Process results as they complete
@@ -581,8 +628,18 @@ def update_all_objectives(
                 error_lst = result[0]
                 for inf in error_lst:
                     error_set.add(inf)
-            progress.update(task, advance=1)
-
+            progress.update(process_task, advance=1)
+    save_task = progress.add_task("[yellow]Saving control objectives...", total=len(UPDATED_IMPLEMENTATION_OBJECTIVES))
+    # Process implementations in parallel
+    # Note, not using threadpool executor here due to phantom 500 errors. This is a workaround
+    for obj in UPDATED_IMPLEMENTATION_OBJECTIVES:
+        try:
+            obj.create_or_update()
+            progress.update(save_task, advance=1)
+        except APIInsertionError as e:
+            error_set.add(f"Failed to create Implementation Objective: {e}")
+        except APIUpdateError as e:
+            error_set.add(f"Failed to update Implementation Objective: {e}")
     return error_set
 
 
@@ -673,7 +730,9 @@ def gen_filtered_records(
     else:
         try:
             control_label = next(
-                dat for dat in part_mapper_rev4.data if dat.get("Oscal Control ID") == security_control.controlId
+                dat
+                for dat in part_mapper_rev4.data
+                if dat.get("Oscal Control ID") == security_control.controlId.lower()
             ).get("CONTROLLABEL")
         except StopIteration:
             control_label = None
@@ -714,15 +773,20 @@ def process_single_record(**kwargs) -> Tuple[List[str], Optional[ImplementationO
     mapped_objectives: List[ControlObjective] = []
     result = None
     parts = []
-    key = record["cis"]["control_id"]
+    # Note: The Control ID from the CIS/CRM can be in non-standard formats, as compared to the example sheet on fedramp.
     if version == "rev5":
+        key = record["cis"]["control_id"].replace(" ", "")
         source = part_mapper_rev5.find_by_source(key)
     else:
+        key = record["cis"]["control_id"]
         source = part_mapper_rev4.find_by_source(key)
         if parts := part_mapper_rev4.find_sub_parts(key):
             for part in parts:
                 try:
-                    mapped_objectives.append(next(obj for obj in control_objectives if obj.name == part))
+                    if version == "rev5":
+                        mapped_objectives.append(next(obj for obj in control_objectives if obj.name == part))
+                    else:
+                        mapped_objectives.append(next(obj for obj in control_objectives if obj.otherId == part))
                 except StopIteration:
                     errors.append(f"Unable to find part {part} for control {key}")
     if not source and not parts:
@@ -733,7 +797,7 @@ def process_single_record(**kwargs) -> Tuple[List[str], Optional[ImplementationO
             objective = next(
                 obj
                 for obj in control_objectives
-                if obj.otherId == source and version == "rev5" or obj.name == source and version == "rev4"
+                if (obj.otherId == source and version in ["rev5", "rev4"]) or (obj.name == source and version == "rev4")
             )
             mapped_objectives.append(objective)
         except StopIteration:
@@ -741,7 +805,7 @@ def process_single_record(**kwargs) -> Tuple[List[str], Optional[ImplementationO
             errors.append(f"Unable to find objective for control {key} ({source})")
 
     if mapped_objectives:
-        result = update_imp_objective(
+        update_imp_objective(
             leverage_auth_id=leveraged_auth_id,
             existing_imp_obj=existing_objectives,
             imp=implementation,
@@ -762,43 +826,58 @@ def parse_crm_worksheet(file_path: click.Path, crm_sheet_name: str, version: Lit
     :return: Formatted CRM content
     :rtype: dict
     """
+    pd = get_pandas()
+    logger.info("Parsing CRM worksheet...")
     formatted_crm = {}
 
     if not crm_sheet_name:
         return formatted_crm
-    import pandas as pd  # Optimize import performance
 
-    if version == "rev5":
-        skip_rows = 2
-    else:
-        skip_rows = 3
+    # Value for rev4
+    skip_rows = 3
 
-    data = pd.read_excel(
+    original_data = pd.read_excel(
         str(file_path),
         sheet_name=crm_sheet_name,
-        skiprows=skip_rows,
-        usecols=[
-            CONTROL_ID,
-            "Can Be Inherited from CSP",
-            "Specific Inheritance and Customer Agency/CSP Responsibilities",
-        ],
     )
 
+    # find index of row where the first column == Control ID
+    skip_rows = determine_skip_row(original_df=original_data, text_to_find=CONTROL_ID, original_skip=skip_rows)
+
+    logger.debug(f"Skipping {skip_rows} rows in CRM worksheet")
+
+    # only use thse coloumns
+    usecols = [
+        CONTROL_ID,
+        "Can Be Inherited from CSP",
+        "Specific Inheritance and Customer Agency/CSP Responsibilities",
+    ]
+
+    try:
+        # Reindex the dataframe and skip some rows
+        data = original_data.iloc[skip_rows:].reset_index(drop=True)
+        data.columns = usecols
+    except KeyError as e:
+        error_and_exit(f"KeyError: {e} - One or more columns specified in usecols are not found in the dataframe.")
+    except Exception as e:
+        error_and_exit(f"An error occurred: {e}")
     # Filter rows where "Can Be Inherited from CSP" is not equal to "No"
     exclude_no = data[data[CAN_BE_INHERITED_CSP] != "No"]
 
     # Iterate through each row and add to the dictionary
     for _, row in exclude_no.iterrows():
         control_id = row[CONTROL_ID]
+        if version == "rev5":
+            control_id = row[CONTROL_ID].replace(" ", "")
 
         # Convert camel case to snake case, remove special characters, and convert to lowercase
         clean_control_id = re.sub(r"\W+", "", control_id)
         clean_control_id = re.sub("([a-z0-9])([A-Z])", r"\1_\2", clean_control_id).lower()
 
         # Use clean_control_id as the key to avoid overwriting
-        formatted_crm[clean_control_id] = {
-            "control_id": clean_control_id,
-            "control_id_original": control_id,
+        formatted_crm[control_id] = {
+            "control_id": control_id,
+            "clean_control_id": clean_control_id,
             "regscale_control_id": transform_control(control_id),
             "can_be_inherited_from_csp": row[CAN_BE_INHERITED_CSP],
             "specific_inheritance_and_customer_agency_csp_responsibilities": row[
@@ -818,10 +897,15 @@ def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
     :return: Formatted CIS content
     :rtype: dict
     """
-    import pandas as pd  # Optimize import performance
-
+    pd = get_pandas()
+    logger.info("Parsing CIS worksheet...")
+    skip_rows = 2
     # Parse the worksheet named 'CIS GovCloud U.S.+DoD (H)', skipping the initial rows
-    cis_df = pd.read_excel(file_path, sheet_name=cis_sheet_name, skiprows=2)
+    original_cis = pd.read_excel(file_path, sheet_name=cis_sheet_name)
+
+    skip_rows = determine_skip_row(original_df=original_cis, text_to_find=CONTROL_ID, original_skip=skip_rows)
+
+    cis_df = original_cis.iloc[skip_rows:].reset_index(drop=True)
 
     # Set the appropriate headers
     cis_df.columns = cis_df.iloc[0]
@@ -918,27 +1002,81 @@ def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
     return {result["control_id"]: result for result in results}
 
 
+def determine_skip_row(original_df: "pd.DataFrame", text_to_find: str, original_skip: int):
+    """
+    Function to determine the row to skip when parsing a worksheet
+
+    :param pd.DataFrame original_df: The original dataframe to search
+    :param str text_to_find: The text to find
+    :param int original_skip: The original row to skip
+    :return: The row to skip
+    :rtype: int
+    """
+    skip_rows = original_skip
+    for idx, row in original_df.iterrows():
+        if row.iloc[0] == text_to_find:
+            skip_rows = idx + 1
+            break
+    return skip_rows
+
+
+def _drop_rows_nan(instructions_df: "pd.DataFrame") -> "pd.DataFrame":
+    """
+    Drop any row with nan and every row after it
+
+    :param pd.DataFrame instructions_df: The instructions dataframe to process
+    :return: The processed dataframe
+    :rtype: pd.DataFrame
+    """
+    # Find the first row containing any NaN value
+    first_nan_index = None
+    for i in range(len(instructions_df)):
+        if instructions_df.iloc[i].isna().any():
+            first_nan_index = i
+            break
+
+    # If a row with NaN is found, keep only rows before it
+    if first_nan_index is not None:
+        instructions_df = instructions_df.iloc[:first_nan_index]
+    else:
+        # Otherwise, just drop any rows with NaN values as before
+        instructions_df = instructions_df.dropna()
+    return instructions_df
+
+
 def parse_instructions_worksheet(
-    file_path: click.Path, version: Literal["rev4", "rev5"], instructions_sheet_name: str = "Instructions"
+    df: "pd.DataFrame",
+    version: Literal["rev4", "rev5"],
+    instructions_sheet_name: str = "Instructions",
 ) -> list[dict]:
     """
     Function to parse the instructions sheet from the FedRAMP Rev5 CIS/CRM workbook
 
-    :param click.Path file_path: The file path to the FedRAMP CIS CRM workbook
+    :param pd.DataFrame df: The dataframe to parse
     :param Literal["rev4", "rev5"] version: The version of the FedRAMP CIS CRM workbook
     :param str instructions_sheet_name: The name of the instructions sheet to parse, defaults to "Instructions"
     :return: List of formatted instructions content as a dictionary
     :rtype: list[dict]
     """
-    import pandas as pd  # Optimize import performance
-
-    instructions_df = pd.read_excel(str(file_path), sheet_name=instructions_sheet_name, skiprows=2)
+    pd = get_pandas()
+    df = df[instructions_sheet_name].iloc[2:]
+    instructions_df = df.dropna(axis=1, how="all")
 
     if version == "rev5":
+        relevant_columns = [CSP, SYSTEM_NAME, "System Identifier", IMPACT_LEVEL]
         # Set the appropriate headers
-        instructions_df.columns = instructions_df.iloc[0]
-        instructions_df = instructions_df[1:]
-        relevant_columns = [SYSTEM_NAME, CSP, "System Identifier", IMPACT_LEVEL]
+        # Find the row with the CSP column e.g. "System Name (CSP to complete all cells)"
+        for index in range(len(instructions_df)):
+            # Check if CSP is in the non-NaN values of this row
+            row_values = [val for val in instructions_df.iloc[index].values if not pd.isna(val)]
+            if CSP in row_values:
+                # Keep only columns that have non-NaN values in this row
+                non_nan_cols = instructions_df.columns[~instructions_df.iloc[index].isna()]
+                instructions_df = instructions_df.loc[:, non_nan_cols]
+                instructions_df.columns = relevant_columns
+                instructions_df = instructions_df[index + 1 :]
+                break
+
     else:
         for index in range(len(instructions_df)):
             if CSP in instructions_df.iloc[index].values:
@@ -949,6 +1087,9 @@ def parse_instructions_worksheet(
         relevant_columns = [SYSTEM_NAME, CSP, IMPACT_LEVEL]
     try:
         instructions_df = instructions_df[relevant_columns]
+        # drop any row with nan
+        instructions_df = _drop_rows_nan(instructions_df)
+
     except KeyError:
         error_and_exit(
             f"Unable to find the relevant columns in the Instructions worksheet. Do you have the correct "
@@ -959,8 +1100,91 @@ def parse_instructions_worksheet(
     return instructions_df.to_dict(orient="records")
 
 
+def update_responsiblity_text():
+    """
+    Update the implementation responsibility texts from the objective data
+    """
+    with ThreadPoolExecutor() as executor:
+        executor.map(_update_imp_responsibility, EXISTING_IMPLEMENTATIONS.values())
+
+
+def _update_imp_responsibility(imp: ControlImplementation):
+    """
+    Update the implementation responsibility text for a given implementation
+
+    :param ControlImplementation imp: The implementation to update
+    :rtype: None
+    :return: None
+    """
+    # Get relevant objectives and sort them
+    objs = _get_sorted_objectives(imp.id)
+
+    # Generate formatted responsibility texts
+    customer_text = _format_responsibility_text(objs, "customerResponsibility")
+    cloud_text = _format_responsibility_text(objs, "cloudResponsibility")
+
+    # Update implementation if we have content
+    if customer_text or cloud_text:
+        _save_implementation_text(imp, customer_text, cloud_text)
+
+
+def _get_sorted_objectives(imp_id: int) -> List[ImplementationObjective]:
+    """
+    Get relevant objectives sorted by notes field
+
+    :param int imp_id: The implementation ID to filter objectives by
+    :rtype: list
+    :return: Sorted list of objectives
+    :rtype: List[ImplementationObjective]
+    """
+    objs = [obj for obj in UPDATED_IMPLEMENTATION_OBJECTIVES if obj.implementationId == imp_id]
+    return sorted(objs, key=lambda x: x.notes)
+
+
+def _format_responsibility_text(objs: list, resp_attr: str) -> str:
+    """
+    Format responsibility text for the given objects and attribute
+
+    :param list objs: The list of objects to format
+    :param str resp_attr: The attribute to format
+    :rtype: str
+    :return: Formatted text
+    """
+    text = ""
+    multi_part = len(objs) > 1
+
+    for obj in objs:
+        resp_text = getattr(obj, resp_attr, "")
+        if resp_text:
+            if multi_part:
+                text += f"<p>part: {obj.notes}</p>"
+            text += f"<p>{resp_text}</p>"
+
+    return text
+
+
+def _save_implementation_text(imp: ControlImplementation, customer_text: str, cloud_text: str):
+    """
+    Save the implementation texts and update parameters
+
+    :param ControlImplementation imp: The implementation to save
+    :param str customer_text: The customer responsibility text
+    :param str cloud_text: The cloud responsibility text
+    :rtype: None
+    :return: None
+    """
+    imp.customerImplementation = customer_text
+    imp.cloudImplementation = cloud_text
+
+    # Update parameters in background thread
+    _spin_off_thread(parameter_merge, imp.id, imp.controlID)
+
+    # Save implementation changes
+    imp.save()
+
+
 def parse_and_map_data(
-    leveraged_auth_id: int, api: Api, ssp_id: int, cis_data: dict, crm_data: dict, version: Literal["rev5", "rev4"]
+    leveraged_auth_id: int, api: Api, cis_data: dict, crm_data: dict, version: Literal["rev5", "rev4"]
 ) -> None:
     """
     Function to parse and map data from RegScale and the workbook.
@@ -974,14 +1198,14 @@ def parse_and_map_data(
     :rtype: None
     """
     with progress:
-        implementations = get_all_imps(api=api, ssp_id=ssp_id, cis_data=cis_data, version=version)
+        get_all_imps(api=api, cis_data=cis_data, version=version)
         error_set = update_all_objectives(
             leveraged_auth_id=leveraged_auth_id,
             cis_data=cis_data,
             crm_data=crm_data,
-            control_implementations=implementations,
             version=version,
         )
+        update_responsiblity_text()
 
     report(error_set)
 
@@ -1032,12 +1256,171 @@ def rev_4_map(control_id: str) -> Optional[str]:
         return f"{base_id}{f'.{letter}' if letter else ''}"
 
 
+def build_implementations_dict(security_plan_id) -> None:
+    """
+    Save the implementations to a dictionary
+
+    :param int security_plan_id: The security plan id
+    :rtype: None
+    :return: None
+    """
+    logger.info("Saving to an implementation dictionary ..")
+    imps = ControlImplementation.get_list_by_plan(security_plan_id)
+    for imp in imps:
+        EXISTING_IMPLEMENTATIONS[imp.controlID] = imp
+    logger.debug("Built %s implementations", len(imps))
+
+
+def create_new_security_plan(profile_id: int, system_name: str):
+    """
+    Create a new FedRamp security plan and map controls based on the profile id.
+
+    :param int profile_id: The profile id to map controls from
+    :param str system_name: The system name to create the security plan for
+    :rtype: SecurityPlan
+    :return: The created security plan
+    """
+    compliance_settings = ComplianceSettings.get_by_current_tenant()
+    try:
+        compliance_setting = next(
+            (
+                setting.id
+                for setting in compliance_settings
+                if setting and setting.title == "FedRAMP Compliance Setting"
+            ),
+            2,
+        )
+    except TypeError:
+        compliance_setting = 2
+    existing_plans = SecurityPlan.get_list()
+    existing_plan = [plan for plan in existing_plans if plan and plan.systemName == system_name]
+    if not existing_plan:
+        profile = Profile.get_object(profile_id)
+        if not profile:
+            error_and_exit("Unable to find the profile with the given ID, please try again")
+        logger.info(f"Loading Profile Mappings from profile #{profile.id} - {profile.name}..")
+        ret = SecurityPlan(
+            **{
+                "status": "Under Development",
+                "systemType": "Major Application",
+                "systemName": system_name,
+                "users": 0,
+                "privilegedUsers": 0,
+                "usersMFA": 0,
+                "privilegedUsersMFA": 0,
+                "internalUsers": 0,
+                "externalUsers": 0,
+                "internalUsersFuture": 0,
+                "externalUsersFuture": 0,
+                "hva": False,
+                "isPublic": True,
+                "bModelSaaS": False,
+                "bModelPaaS": False,
+                "bModelIaaS": False,
+                "bModelOther": False,
+                "otherModelRemarks": "",
+                "bDeployPrivate": False,
+                "bDeployPublic": False,
+                "bDeployGov": False,
+                "bDeployHybrid": False,
+                "bDeployOther": False,
+                "fedrampDateSubmitted": "",
+                "fedrampDateAuthorized": "",
+                "defaultAssessmentDays": 0,
+                "complianceSettingsId": compliance_setting,
+            }
+        ).create()
+        logger.info(f"Created the new Security Plan as ID# {ret.id}")
+        logger.info("Building the implementations from the profile mappings ..")
+        Profile.apply_profile(ret.id, "securityplans", profile_id, True)
+        build_implementations_dict(security_plan_id=ret.id)
+
+    else:
+        ret = next((plan for plan in existing_plan), None)
+        existing_imps = ControlImplementation.get_list_by_plan(ret.id)
+        for imp in existing_imps:
+            EXISTING_IMPLEMENTATIONS[imp.controlID] = imp
+
+    if ret is None:
+        raise ValueError("Unable to create a new security plan.")
+
+    if not EXISTING_IMPLEMENTATIONS:
+        # We must have some implementations, build them if empty.
+        Profile.apply_profile(ret.id, "securityplans", profile_id, True)
+        build_implementations_dict(security_plan_id=ret.id)
+
+    return ret
+
+
+def parameter_merge(implementation_id: int, security_control_id: int):
+    """
+    Merge parameters for a given implementation ID and security control ID.
+
+    :param int implementation_id: The implementation ID
+    :param int security_control_id: The security control ID
+    :rtype: None
+    """
+    parameters = Parameter.merge_parameters(implementation_id, security_control_id)
+    for param in parameters:
+        param.create()
+
+
+def objective_merge(implementation_id: int, security_control_id: int):
+    """
+    Merge objectives for a given implementation ID and security control ID.
+
+    :param int implementation_id: The implementation ID
+    :param int security_control_id: The security control ID
+    :rtype: None
+    """
+    imp_objectives = ImplementationObjective.merge_objectives(implementation_id, security_control_id)
+    for obj in imp_objectives:
+        obj.create()
+
+
+def _spin_off_thread(function: Callable[..., T], *args: Any) -> Thread:
+    """
+    Spin off a thread to run the function with the given arguments.
+
+    :param function: The function to run
+    :param args: The arguments to pass to the function
+    :return: The thread object
+    """
+    thread = Thread(target=function, args=args)
+    thread.start()
+    return thread
+
+
+def _check_sheet_names_exist(
+    file_path: click.Path, cis_sheet_name: str, crm_sheet_name: str
+) -> dict[str, "pd.DataFrame"]:
+    """
+    Check if the sheet names exist in the workbook.
+
+    :param click.Path file_path: The file path to the FedRAMP CIS CRM workbook
+    :param str cis_sheet_name: The name of the CIS sheet to check
+    :param str crm_sheet_name: The name of the CRM sheet to check
+    :raises SystemExit: If the sheet names do not exist
+    :rtype: dict[str, pd.DataFrame]
+    :return: A dictionary of dataframes for each sheet
+    """
+    pd = get_pandas()
+
+    df = pd.read_excel(file_path, sheet_name=None)
+    sheet_names = df.keys()
+    if cis_sheet_name not in sheet_names:
+        error_and_exit(f"The CIS sheet name '{cis_sheet_name}' does not exist in the workbook.")
+    if crm_sheet_name and crm_sheet_name not in sheet_names:
+        error_and_exit(f"The CRM sheet name '{crm_sheet_name}' does not exist in the workbook.")
+    return df
+
+
 def parse_and_import_ciscrm(
     file_path: click.Path,
     version: Literal["rev4", "rev5", "4", "5"],
     cis_sheet_name: str,
-    crm_sheet_name: str,
-    regscale_ssp_id: int,
+    crm_sheet_name: Optional[str],
+    profile_id: int,
     leveraged_auth_id: int = 0,
 ) -> None:
     """
@@ -1046,17 +1429,17 @@ def parse_and_import_ciscrm(
     :param click.Path file_path: The file path to the FedRAMP CIS CRM .xlsx file
     :param Literal["rev4", "rev5"] version: FedRAMP revision version
     :param str cis_sheet_name: CIS sheet name in the FedRAMP CIS CRM .xlsx to parse
-    :param str crm_sheet_name: CRM sheet name in the FedRAMP CIS CRM .xlsx to parse
-    :param int regscale_ssp_id: The ID number from RegScale of the System Security Plan
+    :param Optional[str] crm_sheet_name: CRM sheet name in the FedRAMP CIS CRM .xlsx to parse
+    :param int profile_id: The ID number from RegScale of the RegScale Profile to generate the control mapping
     :param int leveraged_auth_id: RegScale Leveraged Authorization ID #, if none provided, one will be created
     :raises ValueError: If the SSP with the given ID is not found in RegScale
     :rtype: None
     """
+    global REGSCALE_SSP_ID  # Declare that you're modifying the global variable
     sys_name_key = "System Name"
     api = Api()
-    ssp: SecurityPlan = SecurityPlan.get_object(regscale_ssp_id)
-    if not ssp:
-        raise ValueError(f"SSP with ID {regscale_ssp_id} not found in RegScale.")
+
+    df = _check_sheet_names_exist(file_path, cis_sheet_name, crm_sheet_name)
 
     if "5" in version:
         version = "rev5"
@@ -1065,12 +1448,25 @@ def parse_and_import_ciscrm(
         version = "rev4"
         part_mapper_rev4.load_fedramp_version_4_mapping()
     # parse the instructions worksheet to get the csp name, system name, and other data
-    instructions_data = parse_instructions_worksheet(file_path=file_path, version=version)  # type: ignore
+    instructions_data = parse_instructions_worksheet(df=df, version=version)  # type: ignore
 
     # get the system names from the instructions data by dropping any non-string values
-    system_names = [entry[sys_name_key] for entry in instructions_data if isinstance(entry[sys_name_key], str)]
+
+    system_names = [
+        entry[sys_name_key]
+        for entry in instructions_data
+        if isinstance(entry[sys_name_key], str) and cis_sheet_name in entry[sys_name_key].lower()
+    ]
+    if not system_names:
+        system_names = [entry[sys_name_key] for entry in instructions_data if isinstance(entry[sys_name_key], str)]
     name_match: str = system_names[0]
 
+    # create the new security plan
+    ssp: SecurityPlan = create_new_security_plan(profile_id=profile_id, system_name=name_match)
+    REGSCALE_SSP_ID = ssp.id
+
+    if not ssp:
+        raise ValueError("Unable to create a new SSP.")
     # update the instructions data to the matched system names
     instructions_data = [
         (
@@ -1091,9 +1487,7 @@ def parse_and_import_ciscrm(
     cis_data = parse_cis_worksheet(file_path=file_path, cis_sheet_name=cis_sheet_name)
     crm_data = {}
     if crm_sheet_name:
-        crm_data = parse_crm_worksheet(
-            file_path=file_path, crm_sheet_name=crm_sheet_name, version=version  # type: ignore
-        )
+        crm_data = parse_crm_worksheet(file_path=file_path, crm_sheet_name=crm_sheet_name, version=version)  # type: ignore
     if leveraged_auth_id == 0:
         auths = LeveragedAuthorization.get_all_by_parent(ssp.id)
         if auths:
@@ -1101,7 +1495,7 @@ def parse_and_import_ciscrm(
         else:
             leveraged_auth_id = new_leveraged_auth(
                 ssp=ssp,
-                user_id=api.config["userId"],
+                user_id=api.app.config["userId"],
                 instructions_data=instructions_data,
                 version=version,  # type: ignore
             )
@@ -1109,7 +1503,6 @@ def parse_and_import_ciscrm(
     parse_and_map_data(
         leveraged_auth_id=leveraged_auth_id,
         api=api,
-        ssp_id=regscale_ssp_id,
         cis_data=cis_data,
         crm_data=crm_data,
         version=version,  # type: ignore
@@ -1118,7 +1511,7 @@ def parse_and_import_ciscrm(
     # upload workbook to the SSP
     File.upload_file_to_regscale(
         file_name=str(file_path),
-        parent_id=regscale_ssp_id,
+        parent_id=ssp.id,
         parent_module="securityplans",
         api=api,
     )