regscale-cli 6.24.0.1__py3-none-any.whl → 6.25.0.1__py3-none-any.whl

regscale/core/app/internal/evidence.py

@@ -11,21 +11,23 @@ import os
 import shutil
 import zipfile
 from datetime import datetime
+from logging import getLogger
+from pathlib import Path
 from typing import Tuple
 
 import click  # type: ignore
 import pdfplumber  # type: ignore
 from docx import Document  # type: ignore
-from pathlib import Path
 from rich.progress import Progress, TaskID
 
 from regscale.core.app.api import Api
 from regscale.core.app.application import Application
-from regscale.core.app.logz import create_logger
 from regscale.core.app.utils.app_utils import check_file_path, create_progress_object, error_and_exit
 from regscale.models.app_models.click import regscale_ssp_id
 from regscale.models.regscale_models import Assessment, File, Project, SecurityPlan
 
+logger = getLogger("regscale")
+
 
 @click.group()
 def evidence():
@@ -64,74 +66,60 @@ def run_evidence_collection():
     app = Application()
     api = Api()
     config = app.config
+
+    logger.info("Starting evidence collection process")
+
     check_file_path("./static")
     progress = create_progress_object()
     with progress:
-        task0 = progress.add_task("[white]Setting evidence folder directory variables...", total=3)
+        task1 = progress.add_task("[white]Initializing evidence collection...", total=4)
         # call function to define variable for use outside of function
         evidence_folder, dir_name, new_cwd = set_directory_variables(
-            task=task0, evidence_folder=config["evidenceFolder"], progress=progress
+            task=task1, evidence_folder=config["evidenceFolder"], progress=progress
         )
 
-        task1 = progress.add_task("[white]Building a required documents list from config.json...", total=3)
         # call function to define variable for use outside of function
         required_docs, document_list = parse_required_docs(
             evidence_folder=evidence_folder, task=task1, progress=progress
         )
 
-        task2 = progress.add_task("[white]Calculating files last modified times...", total=5)
         # call function to define variable for use outside of function
-        times = get_doc_timestamps(evidence_folder=new_cwd, directory=dir_name, task=task2, progress=progress)
+        times = get_doc_timestamps(evidence_folder=new_cwd, directory=dir_name, task=task1, progress=progress)
 
-        task3 = progress.add_task("[white]Building a required texts list from config.json...", total=3)
         # call function to define variable for use outside of function
-        texts = set_required_texts(evidence_folder=evidence_folder, task=task3, progress=progress)
-
-        task4 = progress.add_task("[white]Searching evidence folder for required files...", total=4)
+        texts = set_required_texts(evidence_folder=evidence_folder, task=task1, progress=progress)
 
         # call function to define variable for use outside of function
-        folders = find_required_files_in_folder(evidence_folder=new_cwd, task=task4, progress=progress)
+        folders = find_required_files_in_folder(evidence_folder=new_cwd, task=task1, progress=progress)
 
-        task5 = progress.add_task("[white]Searching for digital signatures in documents...", total=2)
+        task2 = progress.add_task("[white]Analyzing documents and content...", total=6)
 
         # call function to define variable for use outside of function
         sig_results = signature_assessment_results(
-            directory=folders, r_docs=required_docs, task=task5, progress=progress
+            directory=folders, r_docs=required_docs, task=task2, progress=progress
         )
 
-        task6 = progress.add_task("[white]Testing if required documents are present...", total=2)
-
         # call function to define variable for use outside of function
         doc_results = document_assessment_results(
-            directory=folders, documents=document_list, task=task6, progress=progress
+            directory=folders, documents=document_list, task=task2, progress=progress
         )
 
-        task7 = progress.add_task("[white]Extracting texts from required files...", total=4)
-
         # call function to define variable for use outside of function
-        file_texts = parse_required_text_from_files(evidence_folder=new_cwd, task=task7, progress=progress)
-
-        task8 = progress.add_task("[white]Searching for required text in parsed documents...", total=2)
+        file_texts = parse_required_text_from_files(evidence_folder=new_cwd, task=task2, progress=progress)
 
         # call function to define variable for use outside of function
-        search_results = text_string_search(f_texts=file_texts, req_texts=texts, task=task8, progress=progress)
-
-        task9 = progress.add_task("[white]Testing if required texts are present", total=2)
+        search_results = text_string_search(f_texts=file_texts, req_texts=texts, task=task2, progress=progress)
 
         # call function to define variable for use outside of function
-        text_results = text_assessment_results(searches=search_results, r_texts=texts, task=task9, progress=progress)
+        text_results = text_assessment_results(searches=search_results, r_texts=texts, task=task2, progress=progress)
 
-        task10 = progress.add_task("[white]Retrieving data from the evidence test projects...", total=3)
+        task3 = progress.add_task("[white]Processing assessment data...", total=4)
 
         # call function to define variable for use outside of function
-        data = gather_test_project_data(api=api, evidence_folder=evidence_folder, task=task10, progress=progress)
-
-        task11 = progress.add_task("[white]Testing file modification times...", total=2)
+        data = gather_test_project_data(api=api, evidence_folder=evidence_folder, task=task3, progress=progress)
 
         # call function to define variable to use outside of function
-        time_results = assess_doc_timestamps(timestamps=times, documents=required_docs, task=task11, progress=progress)
-
-        task12 = progress.add_task("[white]Building assessment report...", total=4)
+        time_results = assess_doc_timestamps(timestamps=times, documents=required_docs, task=task3, progress=progress)
 
         # call function to define variable to use outside of function
         report = assessments_report(
@@ -139,32 +127,66 @@ def run_evidence_collection():
             textres=text_results,
             timeres=time_results,
             sigres=sig_results,
-            task=task12,
+            task=task3,
             progress=progress,
         )
 
-        task13 = progress.add_task("[white]Building assessment results dataframe...", total=4)
-
         # call function to define variable to use outside of function
-        results = build_assessment_dataframe(assessments=report, task=task13, progress=progress)
-
-        task14 = progress.add_task("[white]Calculating assessment score...", total=1)
+        results = build_assessment_dataframe(assessments=report, task=task3, progress=progress)
 
         # call function to define variable for use outside of function
-        score_data = build_score_data(assessments=results, task=task14, progress=progress)
-
-        task15 = progress.add_task("[white]Building a table for the assessment report...", total=4)
+        score_data = build_score_data(assessments=results, task=task3, progress=progress)
 
         # call function to define variable for use outside of function
-        html_output = build_html_table(assessments=report, task=task15, progress=progress)
-
-        task16 = progress.add_task("[white]Creating child assessment based on test results...", total=2)
+        html_output = build_html_table(assessments=report, task=task3, progress=progress)
 
         # call function to create child assessment via POST request
         create_child_assessments(
-            api=api, project_data=data, output=html_output, score_data=score_data, task=task16, progress=progress
+            api=api, project_data=data, output=html_output, score_data=score_data, task=task3, progress=progress
         )
 
+        # Display collected files summary
+        display_collected_files(folders, evidence_folder)
+
+
+def display_collected_files(folders: list[dict], evidence_folder: str) -> None:
+    """
+    Display a summary of collected files to the user
+
+    :param list[dict] folders: List of files found in evidence folder
+    :param str evidence_folder: Path to evidence folder
+    :rtype: None
+    """
+    if not folders:
+        logger.info("No files were collected from the evidence folder.")
+        return
+
+    logger.info("=" * 60)
+    logger.info("EVIDENCE COLLECTION SUMMARY")
+    logger.info("=" * 60)
+    logger.info(f"Evidence folder: {evidence_folder}")
+    logger.info(f"Total files collected: {len(folders)}")
+    logger.info("")
+
+    # Group files by program/folder
+    programs = {}
+    for file_info in folders:
+        program = file_info.get("program", "unknown")
+        filename = file_info.get("file", "unknown")
+        if program not in programs:
+            programs[program] = []
+        programs[program].append(filename)
+
+    # Display files by program
+    for program, files in programs.items():
+        logger.info(f"Program: {program}")
+        logger.info("-" * 40)
+        for file in sorted(files):
+            logger.info(f"  • {file}")
+        logger.info("")
+
+    logger.info("=" * 60)
+
 
 def package_builder(ssp_id: int, path: Path):
     """Function to build a directory of evidence and produce a zip file for extraction and use
@@ -374,7 +396,7 @@ def remove_directory(directory_path: Path) -> None:
     :rtype: None
     """
     shutil.rmtree(directory_path.absolute())
-    create_logger().info("Temporary Evidence directory removed successfully!")
+    logger.info("Temporary Evidence directory removed successfully!")
 
 
 def zip_folder(folder_path: Path, zip_path: Path) -> None:
@@ -397,7 +419,7 @@ def zip_folder(folder_path: Path, zip_path: Path) -> None:
                 # Add the file to the ZIP archive using its relative path
                 zipf.write(file_path, relative_path)  # type: ignore
 
-    create_logger().info("Folder zipped successfully!")
+    logger.info("Folder zipped successfully!")
 
 
 def remove(list_to_review: list) -> list:
@@ -462,7 +484,6 @@ def find_signatures(file: str) -> int:
     import pymupdf
 
     number = 0
-    logger = create_logger()
     # if the file is a pdf document
     if file.endswith(".pdf"):
         try:
@@ -507,20 +528,30 @@ def set_directory_variables(task: TaskID, evidence_folder: str, progress: Progre
     # set evidence folder variable to init.yaml value
     # if evidence folder does not exist then create it so tests will pass
     check_file_path(evidence_folder)
+
     # if evidence folder does not exist or if it is empty then error out
-    if evidence_folder is None or len(os.listdir(evidence_folder)) <= 1:
+    evidence_items = os.listdir(evidence_folder)
+
+    if evidence_folder is None or len(evidence_items) == 0:
         error_and_exit("The directory set to evidenceFolder cannot be found or is empty.")
     else:
         # otherwise change directory to the evidence folder
         os.chdir(evidence_folder)
     progress.update(task, advance=1)
-    # include RegScale projects folder
-    dir_name = [filename for filename in os.listdir(os.getcwd()) if os.path.isdir(os.path.join(os.getcwd(), filename))][
-        0
-    ]
-    progress.update(task, advance=1)
-    # pick up subdirectory under the evidence folder
-    new_cwd = os.getcwd() + os.sep + dir_name
+
+    # include RegScale projects folder or use current directory if no subdirs
+    subdirs = [filename for filename in os.listdir(os.getcwd()) if os.path.isdir(os.path.join(os.getcwd(), filename))]
+
+    if subdirs:
+        # Prefer 'project' directory if it exists, otherwise use the first one
+        if "project" in subdirs:
+            dir_name = "project"
+        else:
+            dir_name = subdirs[0]
+        new_cwd = os.getcwd() + os.sep + dir_name
+    else:
+        dir_name = "evidence"
+        new_cwd = os.getcwd()
     progress.update(task, advance=1)
     # return variables for use outside local scope
     return evidence_folder, dir_name, new_cwd
@@ -543,23 +574,41 @@ def parse_required_docs(evidence_folder: str, task: TaskID, progress: Progress)
     document_list = set()
     progress.update(task, advance=1)
     # open app//evidence//config.json file and read contents
-    with open(f"{evidence_folder}{os.sep}config.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        rules = json.load(json_file)
+    config_file = f"{evidence_folder}{os.sep}config.json"
+    if os.path.exists(config_file):
+        with open(config_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            rules = json.load(json_file)
+            progress.update(task, advance=1)
+            # loop through required document dicts
+            for i in range(len(rules.get("required-documents", []))):
+                # add to a list of dictionaries for parsing
+                required_docs.append(
+                    {
+                        "file-name": rules["required-documents"][i].get("file-name"),
+                        "last-updated-by": rules["required-documents"][i].get("last-updated-by"),
+                        "signatures-required": rules["required-documents"][i].get("signatures-required"),
+                        "signature-count": rules["required-documents"][i].get("signature-count"),
+                    }
+                )
+                # update contents of list if it does not already exist
+                document_list.add(rules["required-documents"][i].get("file-name"))
+    else:
+        # No config file, use default requirements for any files found
         progress.update(task, advance=1)
-        # loop through required document dicts
-        for i in range(len(rules["required-documents"])):
-            # add to a list of dictionaries for parsing
-            required_docs.append(
-                {
-                    "file-name": rules["required-documents"][i].get("file-name"),
-                    "last-updated-by": rules["required-documents"][i].get("last-updated-by"),
-                    "signatures-required": rules["required-documents"][i].get("signatures-required"),
-                    "signature-count": rules["required-documents"][i].get("signature-count"),
-                }
-            )
-            # update contents of list if it does not already exist
-            document_list.add(rules["required-documents"][i].get("file-name"))
+        # Get all files in evidence folder and subfolders
+        for root, dirs, files in os.walk(evidence_folder):
+            for file in files:
+                if not file.startswith(".") and file.lower().endswith((".pdf", ".docx", ".doc", ".txt")):
+                    required_docs.append(
+                        {
+                            "file-name": file,
+                            "last-updated-by": 365,
+                            "signatures-required": False,
+                            "signature-count": 0,
+                        }
+                    )
+                    document_list.add(file)
     progress.update(task, advance=1)
     # return variables for use outside of local scope
     return required_docs, document_list
@@ -585,20 +634,37 @@ def get_doc_timestamps(evidence_folder: str, directory: str, task: TaskID, progr
     # remove any child folders that start with '.'
     new_folders = remove(list_to_review=folders_list)
     progress.update(task, advance=1)
-    # loop through directory listing
-    for folder in new_folders:
-        # get list of files in each folder
-        filelist = os.listdir(os.path.join(evidence_folder, folder))
-        # remove any files that start with '.'
-        remove(list_to_review=filelist)
-        # loop through list of files in each folder
+
+    # Check if there are subdirectories
+    subdirs = [f for f in new_folders if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        # loop through directory listing
+        for folder in subdirs:
+            # get list of files in each folder
+            filelist = os.listdir(os.path.join(evidence_folder, folder))
+            # remove any files that start with '.'
+            filelist = remove(filelist)
+            # loop through list of files in each folder
+            modified_times.extend(
+                {
+                    "program": folder,
+                    "file": filename,
+                    "last-modified": os.path.getmtime(os.path.join(directory, folder, filename)),
+                }
+                for filename in filelist
+            )
+    else:
+        # No subdirectories, process files directly in evidence folder
+        files = [f for f in new_folders if os.path.isfile(os.path.join(evidence_folder, f))]
+        files = remove(files)
         modified_times.extend(
             {
-                "program": folder,
+                "program": "evidence",
                 "file": filename,
-                "last-modified": os.path.getmtime(os.path.join(directory, folder, filename)),
+                "last-modified": os.path.getmtime(os.path.join(evidence_folder, filename)),
             }
-            for filename in filelist
+            for filename in files
         )
     progress.update(task, advance=1)
     # loop through the list of timestamps
@@ -624,17 +690,22 @@ def set_required_texts(evidence_folder: str, task: TaskID, progress: Progress) -
     required_text = set()
     progress.update(task, advance=1)
     # open app//evidence//config.json file and read contents
-    with open(f"{evidence_folder}{os.sep}config.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        rules = json.load(json_file)
-        progress.update(task, advance=1)
-        # create iterator to traverse dictionary
-        for i in range(len(rules["rules-engine"])):
-            # pull out required text to look for from config
-            for items in rules["rules-engine"][i]["text-to-find"]:
-                # exclude duplicate text to search from required text
-                required_text.add(items)
+    config_file = f"{evidence_folder}{os.sep}config.json"
+    if os.path.exists(config_file):
+        with open(config_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            rules = json.load(json_file)
+            progress.update(task, advance=1)
+            # create iterator to traverse dictionary
+            for i in range(len(rules.get("rules-engine", []))):
+                # pull out required text to look for from config
+                for items in rules["rules-engine"][i].get("text-to-find", []):
+                    # exclude duplicate text to search from required text
+                    required_text.add(items)
+    else:
+        # No config file, use default text requirements
         progress.update(task, advance=1)
+        required_text = {"security policy", "risk assessment", "compliance", "control", "audit"}
     # return variable for use outside of local scope
     return required_text
 
@@ -658,17 +729,53 @@ def find_required_files_in_folder(evidence_folder: str, task: TaskID, progress:
     # remove any folders starting with '.' from list
     new_folders_list = remove(folder_list)
     progress.update(task, advance=1)
-    for folder in new_folders_list:
-        # build a list of all files contained in sub-directories
-        filelist = os.listdir(evidence_folder + os.sep + folder)
-        # remove folders and file names that start with a .
-        remove(filelist)
-        dir_list.extend({"program": folder, "file": filename} for filename in filelist)
+
+    # Check if there are subdirectories
+    subdirs = [f for f in new_folders_list if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        for folder in subdirs:
+            # build a list of all files contained in sub-directories
+            filelist = os.listdir(evidence_folder + os.sep + folder)
+            # remove folders and file names that start with a .
+            filelist = remove(filelist)
+            dir_list.extend({"program": folder, "file": filename} for filename in filelist)
+    else:
+        # No subdirectories, process files directly in evidence folder
+        files = [f for f in new_folders_list if os.path.isfile(os.path.join(evidence_folder, f))]
+        files = remove(files)
+        dir_list.extend({"program": "evidence", "file": filename} for filename in files)
     progress.update(task, advance=1)
     # return variable for use outside of local scope
     return dir_list
 
 
+def _create_signature_result(program: str, filename: str, test_name: str, result: bool) -> dict:
+    """Helper function to create signature assessment result"""
+    return {
+        "program": program,
+        "file": filename,
+        "test": test_name,
+        "result": result,
+    }
+
+
+def _assess_signature_requirement(doc_file: dict, required: dict) -> list[dict]:
+    """Helper function to assess signature requirements for a document"""
+    results = []
+
+    if required["signatures-required"] is True:
+        sig_result = find_signatures(doc_file["file"])
+        test_name = "signature-required"
+        result = sig_result == 3
+        results.append(_create_signature_result(doc_file["program"], doc_file["file"], test_name, result))
+    elif required["signatures-required"] is False:
+        test_name = "signature-required (not required)"
+        results.append(_create_signature_result(doc_file["program"], doc_file["file"], test_name, True))
+
+    return results
+
+
 def signature_assessment_results(
     directory: list[dict], r_docs: list[dict], task: TaskID, progress: Progress
 ) -> list[dict]:
@@ -682,52 +789,15 @@ def signature_assessment_results(
     :return: Assessment of signatures
     :rtype: list[dict]
     """
-    # create empty list to hold assessment results
     sig_assessments: list[dict] = []
     progress.update(task, advance=1)
-    # loop through list of found documents in each sub-folder
+
     for doc_file in directory:
         for required in r_docs:
             if doc_file["file"] == required["file-name"]:
-                # if the signatures-required field is set to true
-                if required["signatures-required"] is True:
-                    # run the signature detection function for the file
-                    sig_result = find_signatures(doc_file["file"])
-                    # if the return value is 3 pass the test
-                    if sig_result == 3:
-                        # append a true result for each document tested
-                        sig_assessments.append(
-                            {
-                                "program": doc_file["program"],
-                                "file": doc_file["file"],
-                                "test": "signature-required",
-                                "result": True,
-                            }
-                        )
-                    # if the return value is 1, -1 or 0 fail the test
-                    else:
-                        # append a false result for each document tested
-                        sig_assessments.append(
-                            {
-                                "program": doc_file["program"],
-                                "file": doc_file["file"],
-                                "test": "signature-required",
-                                "result": False,
-                            }
-                        )
-                # if the signatures-required field is set to false
-                if required["signatures-required"] is False:
-                    # append a true result for each document not requiring a signature
-                    sig_assessments.append(
-                        {
-                            "program": doc_file["program"],
-                            "file": doc_file["file"],
-                            "test": "signature-required (not required)",
-                            "result": True,
-                        }
-                    )
+                sig_assessments.extend(_assess_signature_requirement(doc_file, required))
+
     progress.update(task, advance=1)
-    # return variable for use outside of local scope
     return sig_assessments
 
 
@@ -775,6 +845,50 @@ def document_assessment_results(
     return doc_assessments
 
 
+def _extract_docx_text(file_path: str) -> list[str]:
+    """Helper function to extract text from DOCX files"""
+    document = Document(file_path)
+    return [para.text for para in document.paragraphs]
+
+
+def _extract_pdf_text(file_path: str) -> list[str]:
+    """Helper function to extract text from PDF files"""
+    output_text_list: list[str] = []
+    with pdfplumber.open(file_path) as pdf:
+        for page in pdf.pages:
+            text = page.extract_text()
+            if text:  # Only append non-None text
+                output_text_list.append(text)
+    return output_text_list
+
+
+def _process_file_for_text(filename: str, file_path: str, program: str) -> dict | None:
+    """Helper function to process a single file and extract text"""
+    if filename.endswith(".docx"):
+        text = _extract_docx_text(file_path)
+    elif filename.endswith(".pdf"):
+        text = _extract_pdf_text(file_path)
+    else:
+        return None
+
+    return {"program": program, "file": filename, "text": text}
+
+
+def _process_files_in_folder(folder_path: str, program: str) -> list[dict]:
+    """Helper function to process all files in a specific folder"""
+    results = []
+    file_list = os.listdir(folder_path)
+    file_list = remove(file_list)
+
+    for filename in file_list:
+        file_path = os.path.join(folder_path, filename)
+        result = _process_file_for_text(filename, file_path, program)
+        if result:
+            results.append(result)
+
+    return results
+
+
 def parse_required_text_from_files(evidence_folder: str, task: TaskID, progress: Progress) -> list[dict]:
     """
     Parse text from docx/pdf file and hold strings representing required text to test
@@ -785,51 +899,26 @@ def parse_required_text_from_files(evidence_folder: str, task: TaskID, progress:
     :return: Results of text found for the files
     :rtype: list[dict]
     """
-    # create an empty list to hold all strings from parsed documents
     full_text: list[dict] = []
     progress.update(task, advance=1)
-    # build a list of files in the folder
+
     folder_list = os.listdir(evidence_folder)
     progress.update(task, advance=1)
-    # remove all folders that start with '.'
     removed_folders_list = remove(folder_list)
     progress.update(task, advance=1)
-    for folder in removed_folders_list:
-        # create a list of files to iterate through for parsing
-        file_list = os.listdir((os.path.join(evidence_folder, folder)))
-        remove(file_list)
-        # iterate through all files in the list
-        for filename in file_list:
-            # if the filename is a .docx file
-            if filename.endswith(".docx"):
-                # open the Word document to enable parsing
-                document = Document(os.path.join(evidence_folder, folder, filename))
-                output: list[str] = [para.text for para in document.paragraphs]
-                # add each file and the requisite text to the dictionary to test
-                full_text.append({"program": folder, "file": filename, "text": output})
-            elif filename.endswith(".pdf"):
-                # create empty list to hold text per file
-                output_text_list: list[str] = []
-                # open filename with pdfplumber
-                with pdfplumber.open(filename) as pdf:
-                    # set number of pages
-                    pages = pdf.pages
-                    # for each page in the pdf document
-                    for page in pages:
-                        # extract the text
-                        text = page.extract_text()
-                        # write the text to a list
-                        output_text_list.append(text)
-                    # add each file and the requisite text to the dictionary to test
-                    full_text.append(
-                        {
-                            "program": folder,
-                            "file": filename,
-                            "text": output_text_list,
-                        }
-                    )
+
+    # Check if there are subdirectories
+    subdirs = [f for f in removed_folders_list if os.path.isdir(os.path.join(evidence_folder, f))]
+
+    if subdirs:
+        for folder in subdirs:
+            folder_path = os.path.join(evidence_folder, folder)
+            full_text.extend(_process_files_in_folder(folder_path, folder))
+    else:
+        # No subdirectories, process files directly in evidence folder
+        full_text.extend(_process_files_in_folder(evidence_folder, "evidence"))
+
     progress.update(task, advance=1)
-    # return variable for use outside of local scope
     return full_text
 
 
@@ -924,17 +1013,22 @@ def gather_test_project_data(api: Api, evidence_folder: str, task: TaskID, progr
     test_data: list[dict] = []
     progress.update(task, advance=1)
     # test project information created in RegScale UI
-    with open(evidence_folder + os.sep + "list.json", "r", encoding="utf-8") as json_file:
-        # load json object into a readable dictionary
-        lists = json.load(json_file)
-        # loop through projects in the list.json
-        test_data.extend(
-            {
-                "id": lists["parser-list"][i].get("id"),
-                "program": lists["parser-list"][i].get("folder-name"),
-            }
-            for i in range(len(lists["parser-list"]))
-        )
+    list_file = evidence_folder + os.sep + "list.json"
+    if os.path.exists(list_file):
+        with open(list_file, "r", encoding="utf-8") as json_file:
+            # load json object into a readable dictionary
+            lists = json.load(json_file)
+            # loop through projects in the list.json
+            test_data.extend(
+                {
+                    "id": lists["parser-list"][i].get("id"),
+                    "program": lists["parser-list"][i].get("folder-name"),
+                }
+                for i in range(len(lists.get("parser-list", [])))
+            )
+    else:
+        # No list.json, skip project data - evidence collection can work without it
+        test_data = []
     progress.update(task, advance=1)
     # create empty list to hold json response data for each project
     test_info: list[dict] = []
@@ -956,7 +1050,7 @@ def gather_test_project_data(api: Api, evidence_folder: str, task: TaskID, progr
                 }
             )
         else:
-            api.logger.error("Project data retrieval was unsuccessful.")
+            api.logger.warning(f"Project data retrieval was unsuccessful for ID {item['id']}, skipping this project.")
     progress.update(task, advance=1)
     # return variables for use outside of local scope
     return test_info
@@ -1031,19 +1125,8 @@ def assessments_report(
     :rtype: list[dict]
     """
     progress.update(task, advance=1)
-    assessment_report: list[dict] = list(docres)
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(textres))
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(timeres))
-    progress.update(task, advance=1)
-    # append all results to 1 master list
-    assessment_report.extend(iter(sigres))
-    progress.update(task, advance=1)
-    # return variable for use outside of local scope
-    return assessment_report
+    # combine all results into one master list
+    return docres + textres + timeres + sigres
 
 
 def build_assessment_dataframe(assessments: list[dict], task: TaskID, progress: Progress) -> list[dict]:
@@ -1061,6 +1144,11 @@ def build_assessment_dataframe(assessments: list[dict], task: TaskID, progress:
 
     result_df = pd.DataFrame(assessments)
     progress.update(task, advance=1)
+
+    # Check if dataframe is empty
+    if result_df.empty:
+        return []
+
     # fill in NaN cells
     result_df = result_df.fillna(" ")
     progress.update(task, advance=1)
@@ -1148,9 +1236,21 @@ def build_html_table(assessments: list[dict], task: TaskID, progress: Progress)
     import pandas as pd  # Optimize import performance
 
     output_list: list[dict] = []
+
+    # Check if assessments is empty
+    if not assessments:
+        progress.update(task, advance=4)  # Skip all remaining progress updates
+        return output_list
+
     # create a dataframe of a list of dicts
     table_df = pd.DataFrame(data=assessments)
     progress.update(task, advance=1)
+
+    # Check if dataframe is empty or missing required columns
+    if table_df.empty or "program" not in table_df.columns:
+        progress.update(task, advance=3)  # Skip remaining progress updates
+        return output_list
+
     # fill in N/A cells with blank string
     table_df = table_df.fillna(" ")
     progress.update(task, advance=1)
@@ -1197,6 +1297,12 @@ def create_child_assessments(
     # set completion datetime to required format
     completion_date = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
     progress.update(task, advance=1)
+
+    # Check if we have project data to work with
+    if not project_data:
+        progress.update(task, advance=1)
+        return
+
     # loop through test projects and make an API call for each
     for i, project in enumerate(project_data):
         # call score calculation function