regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/core/app/internal/evidence.py

@@ -11,20 +11,23 @@ import os
 import shutil
 import zipfile
 from datetime import datetime
+from logging import getLogger
+from pathlib import Path
 from typing import Tuple
 
 import click  # type: ignore
 import pdfplumber  # type: ignore
 from docx import Document  # type: ignore
-from pathlib import Path
 from rich.progress import Progress, TaskID
 
 from regscale.core.app.api import Api
 from regscale.core.app.application import Application
-from regscale.core.app.logz import create_logger
 from regscale.core.app.utils.app_utils import check_file_path, create_progress_object, error_and_exit
 from regscale.models.app_models.click import regscale_ssp_id
-from regscale.models.regscale_models import Assessment, File, Project, SecurityPlan
+from regscale.models.regscale_models import Assessment, File, Project, SecurityPlan, Evidence, Component
+from regscale.models.regscale_models.control_implementation import ControlImplementation
+
+logger = getLogger("regscale")
 
 
 @click.group()
@@ -64,74 +67,60 @@ def run_evidence_collection():
     app = Application()
     api = Api()
     config = app.config
+
+    logger.info("Starting evidence collection process")
+
     check_file_path("./static")
     progress = create_progress_object()
     with progress:
-        task0 = progress.add_task("[white]Setting evidence folder directory variables...", total=3)
+        task1 = progress.add_task("[white]Initializing evidence collection...", total=4)
         # call function to define variable for use outside of function
         evidence_folder, dir_name, new_cwd = set_directory_variables(
-            task=task0, evidence_folder=config["evidenceFolder"], progress=progress
+            task=task1, evidence_folder=config["evidenceFolder"], progress=progress
         )
 
-        task1 = progress.add_task("[white]Building a required documents list from config.json...", total=3)
         # call function to define variable for use outside of function
         required_docs, document_list = parse_required_docs(
             evidence_folder=evidence_folder, task=task1, progress=progress
         )
 
-        task2 = progress.add_task("[white]Calculating files last modified times...", total=5)
         # call function to define variable for use outside of function
-        times = get_doc_timestamps(evidence_folder=new_cwd, directory=dir_name, task=task2, progress=progress)
+        times = get_doc_timestamps(evidence_folder=new_cwd, directory=dir_name, task=task1, progress=progress)
 
-        task3 = progress.add_task("[white]Building a required texts list from config.json...", total=3)
         # call function to define variable for use outside of function
-        texts = set_required_texts(evidence_folder=evidence_folder, task=task3, progress=progress)
-
-        task4 = progress.add_task("[white]Searching evidence folder for required files...", total=4)
+        texts = set_required_texts(evidence_folder=evidence_folder, task=task1, progress=progress)
 
         # call function to define variable for use outside of function
-        folders = find_required_files_in_folder(evidence_folder=new_cwd, task=task4, progress=progress)
+        folders = find_required_files_in_folder(evidence_folder=new_cwd, task=task1, progress=progress)
 
-        task5 = progress.add_task("[white]Searching for digital signatures in documents...", total=2)
+        task2 = progress.add_task("[white]Analyzing documents and content...", total=6)
 
         # call function to define variable for use outside of function
         sig_results = signature_assessment_results(
-            directory=folders, r_docs=required_docs, task=task5, progress=progress
+            directory=folders, r_docs=required_docs, task=task2, progress=progress
         )
 
-        task6 = progress.add_task("[white]Testing if required documents are present...", total=2)
-
         # call function to define variable for use outside of function
         doc_results = document_assessment_results(
-            directory=folders, documents=document_list, task=task6, progress=progress
+            directory=folders, documents=document_list, task=task2, progress=progress
         )
 
-        task7 = progress.add_task("[white]Extracting texts from required files...", total=4)
-
         # call function to define variable for use outside of function
-        file_texts = parse_required_text_from_files(evidence_folder=new_cwd, task=task7, progress=progress)
-
-        task8 = progress.add_task("[white]Searching for required text in parsed documents...", total=2)
+        file_texts = parse_required_text_from_files(evidence_folder=new_cwd, task=task2, progress=progress)
 
         # call function to define variable for use outside of function
-        search_results = text_string_search(f_texts=file_texts, req_texts=texts, task=task8, progress=progress)
-
-        task9 = progress.add_task("[white]Testing if required texts are present", total=2)
+        search_results = text_string_search(f_texts=file_texts, req_texts=texts, task=task2, progress=progress)
 
         # call function to define variable for use outside of function
-        text_results = text_assessment_results(searches=search_results, r_texts=texts, task=task9, progress=progress)
+        text_results = text_assessment_results(searches=search_results, r_texts=texts, task=task2, progress=progress)
 
-        task10 = progress.add_task("[white]Retrieving data from the evidence test projects...", total=3)
+        task3 = progress.add_task("[white]Processing assessment data...", total=4)
 
         # call function to define variable for use outside of function
-        data = gather_test_project_data(api=api, evidence_folder=evidence_folder, task=task10, progress=progress)
-
-        task11 = progress.add_task("[white]Testing file modification times...", total=2)
+        data = gather_test_project_data(api=api, evidence_folder=evidence_folder, task=task3, progress=progress)
 
         # call function to define variable to use outside of function
-        time_results = assess_doc_timestamps(timestamps=times, documents=required_docs, task=task11, progress=progress)
-
-        task12 = progress.add_task("[white]Building assessment report...", total=4)
+        time_results = assess_doc_timestamps(timestamps=times, documents=required_docs, task=task3, progress=progress)
 
         # call function to define variable to use outside of function
         report = assessments_report(
@@ -139,32 +128,66 @@ def run_evidence_collection():
             textres=text_results,
             timeres=time_results,
             sigres=sig_results,
-            task=task12,
+            task=task3,
             progress=progress,
         )
 
-        task13 = progress.add_task("[white]Building assessment results dataframe...", total=4)
-
         # call function to define variable to use outside of function
-        results = build_assessment_dataframe(assessments=report, task=task13, progress=progress)
-
-        task14 = progress.add_task("[white]Calculating assessment score...", total=1)
+        results = build_assessment_dataframe(assessments=report, task=task3, progress=progress)
 
         # call function to define variable for use outside of function
-        score_data = build_score_data(assessments=results, task=task14, progress=progress)
-
-        task15 = progress.add_task("[white]Building a table for the assessment report...", total=4)
+        score_data = build_score_data(assessments=results, task=task3, progress=progress)
 
         # call function to define variable for use outside of function
-        html_output = build_html_table(assessments=report, task=task15, progress=progress)
-
-        task16 = progress.add_task("[white]Creating child assessment based on test results...", total=2)
+        html_output = build_html_table(assessments=report, task=task3, progress=progress)
 
         # call function to create child assessment via POST request
         create_child_assessments(
-            api=api, project_data=data, output=html_output, score_data=score_data, task=task16, progress=progress
+            api=api, project_data=data, output=html_output, score_data=score_data, task=task3, progress=progress
         )
 
+        # Display collected files summary
+        display_collected_files(folders, evidence_folder)
+
+
+def display_collected_files(folders: list[dict], evidence_folder: str) -> None:
+    """
+    Display a summary of collected files to the user
+
+    :param list[dict] folders: List of files found in evidence folder
+    :param str evidence_folder: Path to evidence folder
+    :rtype: None
+    """
+    if not folders:
+        logger.info("No files were collected from the evidence folder.")
+        return
+
+    logger.info("=" * 60)
+    logger.info("EVIDENCE COLLECTION SUMMARY")
+    logger.info("=" * 60)
+    logger.info(f"Evidence folder: {evidence_folder}")
+    logger.info(f"Total files collected: {len(folders)}")
+    logger.info("")
+
+    # Group files by program/folder
+    programs = {}
+    for file_info in folders:
+        program = file_info.get("program", "unknown")
+        filename = file_info.get("file", "unknown")
+        if program not in programs:
+            programs[program] = []
+        programs[program].append(filename)
+
+    # Display files by program
+    for program, files in programs.items():
+        logger.info(f"Program: {program}")
+        logger.info("-" * 40)
+        for file in sorted(files):
+            logger.info(f"  • {file}")
+        logger.info("")
+
+    logger.info("=" * 60)
+
 
 def package_builder(ssp_id: int, path: Path):
     """Function to build a directory of evidence and produce a zip file for extraction and use
@@ -176,7 +199,7 @@ def package_builder(ssp_id: int, path: Path):
     app = Application()
     api = Api()
     with create_progress_object() as progress:
-        task = progress.add_task("[white]Building and zipping evidence folder for audit...", total=6)
+        task = progress.add_task("[white]Building and zipping evidence folder for audit...", total=8)
         try:
             # Obtaining MEGA Api for given Organizer Record.
             ssp = SecurityPlan.fetch_mega_api_data(ssp_id)
@@ -199,6 +222,16 @@ def package_builder(ssp_id: int, path: Path):
 
             progress.update(task, advance=1)
 
+            # Process evidence lockers at SSP level
+            process_ssp_evidence_lockers(
+                ssp_id=ssp_id,
+                path=path,
+                module_folder=module_folder,
+                api=api,
+            )
+
+            progress.update(task, advance=1)
+
             # Checking MEGA Api for Attachments at Control level
             process_control_attachments(
                 ssp=ssp,
@@ -209,6 +242,19 @@ def package_builder(ssp_id: int, path: Path):
                 api=api,
                 task=task,
             )
+
+            progress.update(task, advance=1)
+
+            # Process components and their evidence
+            process_components_evidence(
+                ssp_id=ssp_id,
+                path=path,
+                module_folder=module_folder,
+                api=api,
+            )
+
+            progress.update(task, advance=1)
+
             # Creating zip file and removing temporary Evidence Folder
             new_path = Path("./evidence.zip")
             zip_folder(path, new_path)
@@ -327,6 +373,9 @@ def process_control_attachments(
         # Adding any Attachments at Control level to corresponding folder
         _download_control_attachments(control_attachments, api, path, module_folder_name)
 
+        # Process evidence lockers for controls
+        _process_control_evidence_lockers(control_attachments, api, path, module_folder_name)
+
         progress.update(task, advance=1)
 
     else:
@@ -366,6 +415,396 @@ def _download_control_attachments(
             json.dump(f, file_drop, indent=4, separators=(", ", ": "))
 
 
+def _get_control_folder_name(control_attachments: list[dict], control_id: int) -> str | None:
+    """
+    Get the control folder name for a given control ID
+
+    :param list[dict] control_attachments: List of control attachments
+    :param int control_id: Control ID to find folder name for
+    :return: Control folder name or None
+    :rtype: str | None
+    """
+    for f in control_attachments:
+        if f["parentId"] == control_id:
+            return f["controlId"]
+    return None
+
+
+def _download_control_evidence_items(
+    evidence_items: list[dict], control_folder_name: str, path: Path, module_folder_name: str, api: Api
+) -> None:
+    """
+    Download evidence items for a control
+
+    :param list[dict] evidence_items: List of evidence items
+    :param str control_folder_name: Name of the control folder
+    :param Path path: Base path for downloads
+    :param str module_folder_name: Module folder name
+    :param Api api: API object
+    :rtype: None
+    """
+    logger.info(f"Found {len(evidence_items)} evidence items for control {control_folder_name}")
+
+    for evidence_item in evidence_items:
+        file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+        output_path = f"{path}/{module_folder_name}/{control_folder_name}/{file_name}"
+
+        if download_evidence_file(api, evidence_item, output_path):
+            logger.info(f"Downloaded evidence file: {file_name}")
+        else:
+            logger.warning(f"Failed to download evidence file: {file_name}")
+
+
+def _process_control_evidence_lockers(
+    control_attachments: list[dict], api: Api, path: Path, module_folder_name: str
+) -> None:
+    """
+    Process evidence lockers for controls
+
+    :param list[dict] control_attachments: List of control attachments
+    :param Api api: RegScale CLI API object
+    :param Path path: directory for file location
+    :param str module_folder_name: name of the module folder
+    :rtype: None
+    """
+    # Get unique control IDs
+    control_ids = list({f["parentId"] for f in control_attachments})
+
+    for control_id in control_ids:
+        try:
+            # Get evidence from evidence lockers for this control
+            evidence_items = get_evidence_by_control(api, control_id)
+
+            if evidence_items:
+                # Find the control ID for folder naming
+                control_folder_name = _get_control_folder_name(control_attachments, control_id)
+
+                if control_folder_name:
+                    _download_control_evidence_items(evidence_items, control_folder_name, path, module_folder_name, api)
+        except Exception as e:
+            logger.warning(f"Failed to process evidence lockers for control {control_id}: {e}")
+
+
+def get_evidence_by_control(api: Api, control_id: int) -> list[dict]:
+    """
+    Get evidence for a specific control
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int control_id: Control ID
+    :return: List of evidence items
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Evidence model method instead of direct API call
+        evidence_items = Evidence.get_all_by_parent(parent_id=control_id, parent_module="controls")
+        # Convert to dict format for compatibility
+        return [evidence.dict() for evidence in evidence_items]
+    except Exception as e:
+        logger.warning(f"Failed to get evidence for control {control_id}: {e}")
+        return []
+
+
+def get_evidence_by_security_plan(api: Api, ssp_id: int) -> list[dict]:
+    """
+    Get evidence for a specific security plan
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int ssp_id: Security Plan ID
+    :return: List of evidence items
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Evidence model method instead of direct API call
+        evidence_items = Evidence.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
+        # Convert to dict format for compatibility
+        return [evidence.dict() for evidence in evidence_items]
+    except Exception as e:
+        logger.warning(f"Failed to get evidence for security plan {ssp_id}: {e}")
+        return []
+
+
+def get_components_by_ssp(api: Api, ssp_id: int) -> list[dict]:
+    """
+    Get components for a specific security plan
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int ssp_id: Security Plan ID
+    :return: List of active components
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use Component model method instead of direct API call
+        components = Component.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
+        # Filter for active components only and convert to dict format
+        return [comp.dict() for comp in components if comp.status == "Active"]
+    except Exception as e:
+        logger.warning(f"Failed to get components for security plan {ssp_id}: {e}")
+        return []
+
+
+def get_controls_by_parent(api: Api, parent_id: int, parent_module: str) -> list[dict]:
+    """
+    Get controls for a specific parent (SSP or Component)
+
+    :param Api api: RegScale CLI API object (kept for backward compatibility)
+    :param int parent_id: Parent ID
+    :param str parent_module: Parent module (securityplans or components)
+    :return: List of controls
+    :rtype: list[dict]
+    """
+    # Suppress unused parameter warning for backward compatibility
+    _ = api
+
+    try:
+        # Use ControlImplementation model method instead of direct API call
+        controls = ControlImplementation.get_all_by_parent(parent_id=parent_id, parent_module=parent_module)
+        # Convert to dict format for compatibility
+        return [control.dict() for control in controls]
+    except Exception as e:
+        logger.warning(f"Failed to get controls for parent {parent_id} in module {parent_module}: {e}")
+        return []
+
+
+def download_evidence_file(api: Api, evidence_item: dict, output_path: str) -> bool:
+    """
+    Download an evidence file
+
+    :param Api api: RegScale CLI API object
+    :param dict evidence_item: Evidence item data
+    :param str output_path: Path to save the file
+    :return: True if successful, False otherwise
+    :rtype: bool
+    """
+    try:
+        file_data = File.download_file_from_regscale_to_memory(
+            api=api,
+            record_id=evidence_item["parentId"],
+            module=evidence_item["parentModule"],
+            stored_name=evidence_item["trustedStorageName"],
+            file_hash=evidence_item.get("fileHash") or evidence_item.get("shaHash"),
+        )
+
+        if file_data is None:
+            logger.warning(f"No data received for evidence file {evidence_item.get('trustedDisplayName', 'unknown')}")
+            return False
+
+        with open(output_path, "wb") as f:
+            f.write(file_data)
+        return True
+    except Exception as e:
+        logger.warning(f"Failed to download evidence file {evidence_item.get('trustedDisplayName', 'unknown')}: {e}")
+        return False
+
+
+def process_ssp_evidence_lockers(ssp_id: int, path: Path, module_folder: Path, api: Api) -> None:
+    """
+    Process evidence lockers at SSP level
+
+    :param int ssp_id: Security Plan ID
+    :param Path path: directory for file location
+    :param str module_folder_name: name of the module folder
+    :param Path module_folder: path to module folder
+    :param Api api: RegScale CLI API object
+    :rtype: None
+    """
+    try:
+        # Get evidence from evidence lockers for the SSP
+        evidence_items = get_evidence_by_security_plan(api, ssp_id)
+
+        if evidence_items:
+            logger.info(f"Found {len(evidence_items)} evidence items from evidence lockers for SSP {ssp_id}")
+
+            for evidence_item in evidence_items:
+                file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+                output_path = module_folder / file_name
+
+                if download_evidence_file(api, evidence_item, str(output_path)):
+                    logger.info(f"Downloaded evidence file: {file_name}")
+                else:
+                    logger.warning(f"Failed to download evidence file: {file_name}")
+        else:
+            logger.info("No evidence found in evidence lockers for SSP")
+
+    except Exception as e:
+        logger.warning(f"Error processing SSP evidence lockers: {e}")
+
+
+def _download_files_for_parent(
+    parent_id: int, parent_module: str, output_folder: Path, api: Api, module_name: str = None
+) -> None:
+    """
+    Generalized function to download files for any parent module
+
+    :param int parent_id: Parent ID (component, control, etc.)
+    :param str parent_module: Parent module name (components, controls, etc.)
+    :param Path output_folder: Path to output folder
+    :param Api api: API object
+    :param str module_name: Human-readable module name for logging (optional)
+    :rtype: None
+    """
+    if module_name is None:
+        module_name = parent_module
+
+    try:
+        # Use File model method instead of direct API call
+        files_data = File.get_files_for_parent_from_regscale(api=api, parent_id=parent_id, parent_module=parent_module)
+
+        for file_item in files_data:
+            file_name = file_item.trustedDisplayName or f"file_{file_item.id}"
+            output_path = output_folder / file_name
+
+            try:
+                file_data = File.download_file_from_regscale_to_memory(
+                    api=api,
+                    record_id=file_item.id,
+                    module=parent_module,
+                    stored_name=file_item.trustedStorageName,
+                    file_hash=file_item.fileHash or file_item.shaHash,
+                )
+
+                if file_data is None:
+                    logger.warning(f"No data received for {module_name} file {file_name}")
+                    continue
+
+                with open(output_path, "wb") as f:
+                    f.write(file_data)
+                logger.info(f"Downloaded {module_name} file: {file_name}")
+            except Exception as e:
+                logger.warning(f"Failed to download {module_name} file {file_name}: {e}")
+    except Exception as e:
+        logger.warning(f"Failed to get {module_name} files for {parent_module} {parent_id}: {e}")
+
+
+def _download_component_files(component_id: int, component_folder: Path, api: Api) -> None:
+    """
+    Download files directly attached to a component
+
+    :param int component_id: Component ID
+    :param Path component_folder: Path to component folder
+    :param Api api: API object
+    :rtype: None
+    """
+    _download_files_for_parent(
+        parent_id=component_id,
+        parent_module="components",
+        output_folder=component_folder,
+        api=api,
+        module_name="component",
+    )
+
+
+def _download_control_files(control_id: int, control_folder: Path, api: Api) -> None:
+    """
+    Download files for a control
+
+    :param int control_id: Control ID
+    :param Path control_folder: Path to control folder
+    :param Api api: API object
+    :rtype: None
+    """
+    _download_files_for_parent(
+        parent_id=control_id, parent_module="controls", output_folder=control_folder, api=api, module_name="control"
+    )
+
+
+def _download_control_evidence(control_id: int, control_folder: Path, api: Api) -> None:
+    """
+    Download evidence from evidence lockers for a control
+
+    :param int control_id: Control ID
+    :param Path control_folder: Path to control folder
+    :param Api api: API object
+    :rtype: None
+    """
+    evidence_items = get_evidence_by_control(api, control_id)
+
+    if evidence_items:
+        logger.info(f"Found {len(evidence_items)} evidence items for control {control_folder.name}")
+
+        for evidence_item in evidence_items:
+            file_name = evidence_item.get("trustedDisplayName", f"evidence_{evidence_item.get('id', 'unknown')}")
+            output_path = control_folder / file_name
+
+            if download_evidence_file(api, evidence_item, str(output_path)):
+                logger.info(f"Downloaded evidence file: {file_name}")
+            else:
+                logger.warning(f"Failed to download evidence file: {file_name}")
+
+
+def _process_component_controls(component_id: int, component_folder: Path, api: Api) -> None:
+    """
+    Process controls for a component
+
+    :param int component_id: Component ID
+    :param Path component_folder: Path to component folder
+    :param Api api: API object
+    :rtype: None
+    """
+    controls = get_controls_by_parent(api, component_id, "components")
+
+    if controls:
+        logger.info(f"Found {len(controls)} controls for component {component_folder.name}")
+
+        for control in controls:
+            control_id = control.get("id")
+            control_name = control.get("controlId", f"Control_{control_id}")
+
+            # Create control folder within component folder
+            control_folder = component_folder / control_name
+            os.makedirs(control_folder, exist_ok=True)
+
+            # Download control files and evidence
+            _download_control_files(control_id, control_folder, api)
+            _download_control_evidence(control_id, control_folder, api)
+
+
+def process_components_evidence(ssp_id: int, path: Path, module_folder: Path, api: Api) -> None:
+    """
+    Process components and their evidence
+
+    :param int ssp_id: Security Plan ID
+    :param Path path: directory for file location
+    :param Path module_folder: path to module folder
+    :param Api api: RegScale CLI API object
+    :rtype: None
+    """
+    try:
+        # Get components for the SSP
+        components = get_components_by_ssp(api, ssp_id)
+
+        if not components:
+            logger.info("No active components found for SSP")
+            return
+
+        logger.info(f"Found {len(components)} active components for SSP {ssp_id}")
+
+        for component in components:
+            component_id = component.get("id")
+            component_title = component.get("title", f"Component_{component_id}")
+
+            # Create component folder
+            component_folder = module_folder / component_title
+            os.makedirs(component_folder, exist_ok=True)
+
+            # Download component files
+            _download_component_files(component_id, component_folder, api)
+
+            # Process component controls
+            _process_component_controls(component_id, component_folder, api)
+
+    except Exception as e:
+        logger.warning(f"Error processing components evidence: {e}")
+
+
 def remove_directory(directory_path: Path) -> None:
     """
     This function removes a given directory even if files stored there
@@ -374,7 +813,7 @@ def remove_directory(directory_path: Path) -> None:
     :rtype: None
     """
     shutil.rmtree(directory_path.absolute())
-    create_logger().info("Temporary Evidence directory removed successfully!")
+    logger.info("Temporary Evidence directory removed successfully!")
 
 
 def zip_folder(folder_path: Path, zip_path: Path) -> None:
@@ -397,7 +836,7 @@ def zip_folder(folder_path: Path, zip_path: Path) -> None:
                 # Add the file to the ZIP archive using its relative path
                 zipf.write(file_path, relative_path)  # type: ignore
 
-    create_logger().info("Folder zipped successfully!")
+    logger.info("Folder zipped successfully!")
 
 
 def remove(list_to_review: list) -> list:
@@ -462,7 +901,6 @@ def find_signatures(file: str) -> int:
     import pymupdf
 
     number = 0
-    logger = create_logger()
     # if the file is a pdf document
     if file.endswith(".pdf"):
         try:
@@ -507,20 +945,30 @@ def set_directory_variables(task: TaskID, evidence_folder: str, progress: Progre
     # set evidence folder variable to init.yaml value
     # if evidence folder does not exist then create it so tests will pass
     check_file_path(evidence_folder)
+
     # if evidence folder does not exist or if it is empty then error out
-    if evidence_folder is None or len(os.listdir(evidence_folder)) <= 1:
+    evidence_items = os.listdir(evidence_folder)
+
+    if evidence_folder is None or len(evidence_items) == 0:
         error_and_exit("The directory set to evidenceFolder cannot be found or is empty.")
     else:
         # otherwise change directory to the evidence folder
         os.chdir(evidence_folder)
     progress.update(task, advance=1)
-    # include RegScale projects folder
-    dir_name = [filename for filename in os.listdir(os.getcwd()) if os.path.isdir(os.path.join(os.getcwd(), filename))][
-        0
-    ]
-    progress.update(task, advance=1)
-    # pick up subdirectory under the evidence folder
-    new_cwd = os.getcwd() + os.sep + dir_name
+
+    # include RegScale projects folder or use current directory if no subdirs
+    subdirs = [filename for filename in os.listdir(os.getcwd()) if os.path.isdir(os.path.join(os.getcwd(), filename))]
+
+    if subdirs:
+        # Prefer 'project' directory if it exists, otherwise use the first one
+        if "project" in subdirs:
+            dir_name = "project"
+        else:
+            dir_name = subdirs[0]
+        new_cwd = os.getcwd() + os.sep + dir_name
+    else:
+        dir_name = "evidence"
+        new_cwd = os.getcwd()
     progress.update(task, advance=1)
     # return variables for use outside local scope
     return evidence_folder, dir_name, new_cwd
@@ -543,23 +991,41 @@ def parse_required_docs(evidence_folder: str, task: TaskID, progress: Progress)
     document_list = set()
     progress.update(task, advance=1)
     # open app//evidence//config.json file and read contents
-    with open(f"{evidence_folder}{os.sep}config.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        rules = json.load(json_file)
+    config_file = f"{evidence_folder}{os.sep}config.json"
+    if os.path.exists(config_file):
+        with open(config_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            rules = json.load(json_file)
+            progress.update(task, advance=1)
+            # loop through required document dicts
+            for i in range(len(rules.get("required-documents", []))):
+                # add to a list of dictionaries for parsing
+                required_docs.append(
+                    {
+                        "file-name": rules["required-documents"][i].get("file-name"),
+                        "last-updated-by": rules["required-documents"][i].get("last-updated-by"),
+                        "signatures-required": rules["required-documents"][i].get("signatures-required"),
+                        "signature-count": rules["required-documents"][i].get("signature-count"),
+                    }
+                )
+                # update contents of list if it does not already exist
+                document_list.add(rules["required-documents"][i].get("file-name"))
+    else:
+        # No config file, use default requirements for any files found
         progress.update(task, advance=1)
-        # loop through required document dicts
-        for i in range(len(rules["required-documents"])):
-            # add to a list of dictionaries for parsing
-            required_docs.append(
-                {
-                    "file-name": rules["required-documents"][i].get("file-name"),
-                    "last-updated-by": rules["required-documents"][i].get("last-updated-by"),
-                    "signatures-required": rules["required-documents"][i].get("signatures-required"),
-                    "signature-count": rules["required-documents"][i].get("signature-count"),
-                }
-            )
-            # update contents of list if it does not already exist
-            document_list.add(rules["required-documents"][i].get("file-name"))
+        # Get all files in evidence folder and subfolders
+        for root, dirs, files in os.walk(evidence_folder):
+            for file in files:
+                if not file.startswith(".") and file.lower().endswith((".pdf", ".docx", ".doc", ".txt")):
+                    required_docs.append(
+                        {
+                            "file-name": file,
+                            "last-updated-by": 365,
+                            "signatures-required": False,
+                            "signature-count": 0,
+                        }
+                    )
+                    document_list.add(file)
     progress.update(task, advance=1)
     # return variables for use outside of local scope
     return required_docs, document_list
@@ -585,20 +1051,37 @@ def get_doc_timestamps(evidence_folder: str, directory: str, task: TaskID, progr
     # remove any child folders that start with '.'
     new_folders = remove(list_to_review=folders_list)
     progress.update(task, advance=1)
-    # loop through directory listing
-    for folder in new_folders:
-        # get list of files in each folder
-        filelist = os.listdir(os.path.join(evidence_folder, folder))
-        # remove any files that start with '.'
-        remove(list_to_review=filelist)
-        # loop through list of files in each folder
+
+    # Check if there are subdirectories
+    subdirs = [f for f in new_folders if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        # loop through directory listing
+        for folder in subdirs:
+            # get list of files in each folder
+            filelist = os.listdir(os.path.join(evidence_folder, folder))
+            # remove any files that start with '.'
+            filelist = remove(filelist)
+            # loop through list of files in each folder
+            modified_times.extend(
+                {
+                    "program": folder,
+                    "file": filename,
+                    "last-modified": os.path.getmtime(os.path.join(directory, folder, filename)),
+                }
+                for filename in filelist
+            )
+    else:
+        # No subdirectories, process files directly in evidence folder
+        files = [f for f in new_folders if os.path.isfile(os.path.join(evidence_folder, f))]
+        files = remove(files)
         modified_times.extend(
             {
-                "program": folder,
+                "program": "evidence",
                 "file": filename,
-                "last-modified": os.path.getmtime(os.path.join(directory, folder, filename)),
+                "last-modified": os.path.getmtime(os.path.join(evidence_folder, filename)),
             }
-            for filename in filelist
+            for filename in files
         )
     progress.update(task, advance=1)
     # loop through the list of timestamps
@@ -624,17 +1107,22 @@ def set_required_texts(evidence_folder: str, task: TaskID, progress: Progress) -
     required_text = set()
     progress.update(task, advance=1)
     # open app//evidence//config.json file and read contents
-    with open(f"{evidence_folder}{os.sep}config.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        rules = json.load(json_file)
-        progress.update(task, advance=1)
-        # create iterator to traverse dictionary
-        for i in range(len(rules["rules-engine"])):
-            # pull out required text to look for from config
-            for items in rules["rules-engine"][i]["text-to-find"]:
-                # exclude duplicate text to search from required text
-                required_text.add(items)
+    config_file = f"{evidence_folder}{os.sep}config.json"
+    if os.path.exists(config_file):
+        with open(config_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            rules = json.load(json_file)
+            progress.update(task, advance=1)
+            # create iterator to traverse dictionary
+            for i in range(len(rules.get("rules-engine", []))):
+                # pull out required text to look for from config
+                for items in rules["rules-engine"][i].get("text-to-find", []):
+                    # exclude duplicate text to search from required text
+                    required_text.add(items)
+    else:
+        # No config file, use default text requirements
         progress.update(task, advance=1)
+        required_text = {"security policy", "risk assessment", "compliance", "control", "audit"}
     # return variable for use outside of local scope
     return required_text
 
@@ -658,17 +1146,53 @@ def find_required_files_in_folder(evidence_folder: str, task: TaskID, progress:
     # remove any folders starting with '.' from list
     new_folders_list = remove(folder_list)
     progress.update(task, advance=1)
-    for folder in new_folders_list:
-        # build a list of all files contained in sub-directories
-        filelist = os.listdir(evidence_folder + os.sep + folder)
-        # remove folders and file names that start with a .
-        remove(filelist)
-        dir_list.extend({"program": folder, "file": filename} for filename in filelist)
+
+    # Check if there are subdirectories
+    subdirs = [f for f in new_folders_list if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        for folder in subdirs:
+            # build a list of all files contained in sub-directories
+            filelist = os.listdir(evidence_folder + os.sep + folder)
+            # remove folders and file names that start with a .
+            filelist = remove(filelist)
+            dir_list.extend({"program": folder, "file": filename} for filename in filelist)
+    else:
+        # No subdirectories, process files directly in evidence folder
+        files = [f for f in new_folders_list if os.path.isfile(os.path.join(evidence_folder, f))]
+        files = remove(files)
+        dir_list.extend({"program": "evidence", "file": filename} for filename in files)
     progress.update(task, advance=1)
     # return variable for use outside of local scope
     return dir_list
 
 
+def _create_signature_result(program: str, filename: str, test_name: str, result: bool) -> dict:
+    """Helper function to create signature assessment result"""
+    return {
+        "program": program,
+        "file": filename,
+        "test": test_name,
+        "result": result,
+    }
+
+
+def _assess_signature_requirement(doc_file: dict, required: dict) -> list[dict]:
+    """Helper function to assess signature requirements for a document"""
+    results = []
+
+    if required["signatures-required"] is True:
+        sig_result = find_signatures(doc_file["file"])
+        test_name = "signature-required"
+        result = sig_result == 3
+        results.append(_create_signature_result(doc_file["program"], doc_file["file"], test_name, result))
+    elif required["signatures-required"] is False:
+        test_name = "signature-required (not required)"
+        results.append(_create_signature_result(doc_file["program"], doc_file["file"], test_name, True))
+
+    return results
+
+
 def signature_assessment_results(
     directory: list[dict], r_docs: list[dict], task: TaskID, progress: Progress
 ) -> list[dict]:
@@ -682,52 +1206,15 @@ def signature_assessment_results(
     :return: Assessment of signatures
     :rtype: list[dict]
     """
-    # create empty list to hold assessment results
     sig_assessments: list[dict] = []
     progress.update(task, advance=1)
-    # loop through list of found documents in each sub-folder
+
     for doc_file in directory:
         for required in r_docs:
             if doc_file["file"] == required["file-name"]:
-                # if the signatures-required field is set to true
-                if required["signatures-required"] is True:
-                    # run the signature detection function for the file
-                    sig_result = find_signatures(doc_file["file"])
-                    # if the return value is 3 pass the test
-                    if sig_result == 3:
-                        # append a true result for each document tested
-                        sig_assessments.append(
-                            {
-                                "program": doc_file["program"],
-                                "file": doc_file["file"],
-                                "test": "signature-required",
-                                "result": True,
-                            }
-                        )
-                    # if the return value is 1, -1 or 0 fail the test
-                    else:
-                        # append a false result for each document tested
-                        sig_assessments.append(
-                            {
-                                "program": doc_file["program"],
-                                "file": doc_file["file"],
-                                "test": "signature-required",
-                                "result": False,
-                            }
-                        )
-                # if the signatures-required field is set to false
-                if required["signatures-required"] is False:
-                    # append a true result for each document not requiring a signature
-                    sig_assessments.append(
-                        {
-                            "program": doc_file["program"],
-                            "file": doc_file["file"],
-                            "test": "signature-required (not required)",
-                            "result": True,
-                        }
-                    )
+                sig_assessments.extend(_assess_signature_requirement(doc_file, required))
+
     progress.update(task, advance=1)
-    # return variable for use outside of local scope
     return sig_assessments
 
 
@@ -775,6 +1262,50 @@ def document_assessment_results(
     return doc_assessments
 
 
+def _extract_docx_text(file_path: str) -> list[str]:
+    """Helper function to extract text from DOCX files"""
+    document = Document(file_path)
+    return [para.text for para in document.paragraphs]
+
+
+def _extract_pdf_text(file_path: str) -> list[str]:
+    """Helper function to extract text from PDF files"""
+    output_text_list: list[str] = []
+    with pdfplumber.open(file_path) as pdf:
+        for page in pdf.pages:
+            text = page.extract_text()
+            if text:  # Only append non-None text
+                output_text_list.append(text)
+    return output_text_list
+
+
+def _process_file_for_text(filename: str, file_path: str, program: str) -> dict | None:
+    """Helper function to process a single file and extract text"""
+    if filename.endswith(".docx"):
+        text = _extract_docx_text(file_path)
+    elif filename.endswith(".pdf"):
+        text = _extract_pdf_text(file_path)
+    else:
+        return None
+
+    return {"program": program, "file": filename, "text": text}
+
+
+def _process_files_in_folder(folder_path: str, program: str) -> list[dict]:
+    """Helper function to process all files in a specific folder"""
+    results = []
+    file_list = os.listdir(folder_path)
+    file_list = remove(file_list)
+
+    for filename in file_list:
+        file_path = os.path.join(folder_path, filename)
+        result = _process_file_for_text(filename, file_path, program)
+        if result:
+            results.append(result)
+
+    return results
+
+
 def parse_required_text_from_files(evidence_folder: str, task: TaskID, progress: Progress) -> list[dict]:
     """
     Parse text from docx/pdf file and hold strings representing required text to test
@@ -785,51 +1316,26 @@ def parse_required_text_from_files(evidence_folder: str, task: TaskID, progress:
     :return: Results of text found for the files
     :rtype: list[dict]
     """
-    # create an empty list to hold all strings from parsed documents
     full_text: list[dict] = []
     progress.update(task, advance=1)
-    # build a list of files in the folder
+
     folder_list = os.listdir(evidence_folder)
     progress.update(task, advance=1)
-    # remove all folders that start with '.'
     removed_folders_list = remove(folder_list)
     progress.update(task, advance=1)
-    for folder in removed_folders_list:
-        # create a list of files to iterate through for parsing
-        file_list = os.listdir((os.path.join(evidence_folder, folder)))
-        remove(file_list)
-        # iterate through all files in the list
-        for filename in file_list:
-            # if the filename is a .docx file
-            if filename.endswith(".docx"):
-                # open the Word document to enable parsing
-                document = Document(os.path.join(evidence_folder, folder, filename))
-                output: list[str] = [para.text for para in document.paragraphs]
-                # add each file and the requisite text to the dictionary to test
-                full_text.append({"program": folder, "file": filename, "text": output})
-            elif filename.endswith(".pdf"):
-                # create empty list to hold text per file
-                output_text_list: list[str] = []
-                # open filename with pdfplumber
-                with pdfplumber.open(filename) as pdf:
-                    # set number of pages
-                    pages = pdf.pages
-                    # for each page in the pdf document
-                    for page in pages:
-                        # extract the text
-                        text = page.extract_text()
-                        # write the text to a list
-                        output_text_list.append(text)
-                    # add each file and the requisite text to the dictionary to test
-                    full_text.append(
-                        {
-                            "program": folder,
-                            "file": filename,
-                            "text": output_text_list,
-                        }
-                    )
+
+    # Check if there are subdirectories
+    subdirs = [f for f in removed_folders_list if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        for folder in subdirs:
+            folder_path = os.path.join(evidence_folder, folder)
+            full_text.extend(_process_files_in_folder(folder_path, folder))
+    else:
+        # No subdirectories, process files directly in evidence folder
+        full_text.extend(_process_files_in_folder(evidence_folder, "evidence"))
+
     progress.update(task, advance=1)
-    # return variable for use outside of local scope
     return full_text
 
 
@@ -924,17 +1430,22 @@ def gather_test_project_data(api: Api, evidence_folder: str, task: TaskID, progr
     test_data: list[dict] = []
     progress.update(task, advance=1)
     # test project information created in RegScale UI
-    with open(evidence_folder + os.sep + "list.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        lists = json.load(json_file)
-        # loop through projects in the list.json
-        test_data.extend(
-            {
-                "id": lists["parser-list"][i].get("id"),
-                "program": lists["parser-list"][i].get("folder-name"),
-            }
-            for i in range(len(lists["parser-list"]))
-        )
+    list_file = evidence_folder + os.sep + "list.json"
+    if os.path.exists(list_file):
+        with open(list_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            lists = json.load(json_file)
+            # loop through projects in the list.json
+            test_data.extend(
+                {
+                    "id": lists["parser-list"][i].get("id"),
+                    "program": lists["parser-list"][i].get("folder-name"),
+                }
+                for i in range(len(lists.get("parser-list", [])))
+            )
+    else:
+        # No list.json, skip project data - evidence collection can work without it
+        test_data = []
     progress.update(task, advance=1)
     # create empty list to hold json response data for each project
     test_info: list[dict] = []
@@ -956,7 +1467,7 @@ def gather_test_project_data(api: Api, evidence_folder: str, task: TaskID, progr
                 }
             )
         else:
-            api.logger.error("Project data retrieval was unsuccessful.")
+            api.logger.warning(f"Project data retrieval was unsuccessful for ID {item['id']}, skipping this project.")
     progress.update(task, advance=1)
     # return variables for use outside of local scope
     return test_info
@@ -1031,19 +1542,8 @@ def assessments_report(
     :rtype: list[dict]
     """
     progress.update(task, advance=1)
-    assessment_report: list[dict] = list(docres)
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(textres))
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(timeres))
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(sigres))
-    progress.update(task, advance=1)
-    # return variable for use outside of local scope
-    return assessment_report
+    # combine all results into one master list
+    return docres + textres + timeres + sigres
 
 
 def build_assessment_dataframe(assessments: list[dict], task: TaskID, progress: Progress) -> list[dict]:
@@ -1061,6 +1561,11 @@ def build_assessment_dataframe(assessments: list[dict], task: TaskID, progress:
 
     result_df = pd.DataFrame(assessments)
     progress.update(task, advance=1)
+
+    # Check if dataframe is empty
+    if result_df.empty:
+        return []
+
     # fill in NaN cells
     result_df = result_df.fillna(" ")
     progress.update(task, advance=1)
@@ -1148,9 +1653,21 @@ def build_html_table(assessments: list[dict], task: TaskID, progress: Progress)
     import pandas as pd  # Optimize import performance
 
     output_list: list[dict] = []
+
+    # Check if assessments is empty
+    if not assessments:
+        progress.update(task, advance=4)  # Skip all remaining progress updates
+        return output_list
+
     # create a dataframe of a list of dicts
     table_df = pd.DataFrame(data=assessments)
     progress.update(task, advance=1)
+
+    # Check if dataframe is empty or missing required columns
+    if table_df.empty or "program" not in table_df.columns:
+        progress.update(task, advance=3)  # Skip remaining progress updates
+        return output_list
+
     # fill in N/A cells with blank string
     table_df = table_df.fillna(" ")
     progress.update(task, advance=1)
@@ -1197,6 +1714,12 @@ def create_child_assessments(
     # set completion datetime to required format
     completion_date = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
     progress.update(task, advance=1)
+
+    # Check if we have project data to work with
+    if not project_data:
+        progress.update(task, advance=1)
+        return
+
     # loop through test projects and make an API call for each
     for i, project in enumerate(project_data):
         # call score calculation function