contentctl 4.3.2__py3-none-any.whl → 4.3.4__py3-none-any.whl

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -10,7 +10,6 @@ import pathlib
 from tempfile import TemporaryDirectory, mktemp
 from ssl import SSLEOFError, SSLZeroReturnError
 from sys import stdout
-#from dataclasses import dataclass
 from shutil import copyfile
 from typing import Union, Optional
 
@@ -29,7 +28,7 @@ from contentctl.objects.detection import Detection
 from contentctl.objects.base_test import BaseTest
 from contentctl.objects.unit_test import UnitTest
 from contentctl.objects.integration_test import IntegrationTest
-from contentctl.objects.unit_test_attack_data import UnitTestAttackData
+from contentctl.objects.test_attack_data import TestAttackData
 from contentctl.objects.unit_test_result import UnitTestResult
 from contentctl.objects.integration_test_result import IntegrationTestResult
 from contentctl.objects.test_group import TestGroup
@@ -61,13 +60,19 @@ class CleanupTestGroupResults(BaseModel):
 
 class ContainerStoppedException(Exception):
     pass
+class CannotRunBaselineException(Exception):
+    # Support for testing detections with baselines 
+    # does not currently exist in contentctl.
+    # As such, whenever we encounter a detection 
+    # with baselines we should generate a descriptive
+    # exception
+    pass
 
 
 @dataclasses.dataclass(frozen=False)
 class DetectionTestingManagerOutputDto():
     inputQueue: list[Detection] = Field(default_factory=list)
     outputQueue: list[Detection] = Field(default_factory=list)
-    skippedQueue: list[Detection] = Field(default_factory=list)
     currentTestingQueue: dict[str, Union[Detection, None]] = Field(default_factory=dict)
     start_time: Union[datetime.datetime, None] = None
     replay_index: str = "CONTENTCTL_TESTING_INDEX"
@@ -369,12 +374,6 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
                 return
 
             try:
-                # NOTE: (THIS CODE HAS MOVED) we handle skipping entire detections differently than
-                #   we do skipping individual test cases; we skip entire detections by excluding
-                #   them to an entirely separate queue, while we skip individual test cases via the
-                #   BaseTest.skip() method, such as when we are skipping all integration tests (see
-                #   DetectionBuilder.skipIntegrationTests)
-                # TODO: are we skipping by production status elsewhere?
                 detection = self.sync_obj.inputQueue.pop()
                 self.sync_obj.currentTestingQueue[self.get_name()] = detection
             except IndexError:
@@ -647,11 +646,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
         # Set the mode and timeframe, if required
         kwargs = {"exec_mode": "blocking"}
 
-        # Iterate over baselines (if any)
-        for baseline in test.baselines:
-            # TODO: this is executing the test, not the baseline...
-            # TODO: should this be in a try/except if the later call is?
-            self.retry_search_until_timeout(detection, test, kwargs, test_start_time)
+        
 
         # Set earliest_time and latest_time appropriately if FORCE_ALL_TIME is False
         if not FORCE_ALL_TIME:
@@ -662,7 +657,23 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
 
         # Run the detection's search query
         try:
+            # Iterate over baselines (if any)
+            for baseline in detection.baselines:
+                raise CannotRunBaselineException("Detection requires Execution of a Baseline, "
+                                                 "however Baseline execution is not "
+                                                 "currently supported in contentctl. Mark "
+                                                 "this as manual_test.")
             self.retry_search_until_timeout(detection, test, kwargs, test_start_time)
+        except CannotRunBaselineException as e:
+            # Init the test result and record a failure if there was an issue during the search
+            test.result = UnitTestResult()
+            test.result.set_job_content(
+                None,
+                self.infrastructure,
+                TestResultStatus.ERROR,
+                exception=e,
+                duration=time.time() - test_start_time
+            )
         except ContainerStoppedException as e:
             raise e
         except Exception as e:
@@ -1015,18 +1026,15 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
         """
         # Get the start time and compute the timeout
         search_start_time = time.time()
-        search_stop_time = time.time() + self.sync_obj.timeout_seconds
-
-        # We will default to ensuring at least one result exists
-        if test.pass_condition is None:
-            search = detection.search
-        else:
-            # Else, use the explicit pass condition
-            search = f"{detection.search} {test.pass_condition}"
+        search_stop_time = time.time() + self.sync_obj.timeout_seconds        
+        
+        # Make a copy of the search string since we may 
+        # need to make some small changes to it below
+        search = detection.search
 
         # Ensure searches that do not begin with '|' must begin with 'search '
-        if not search.strip().startswith("|"):                                                      # type: ignore
-            if not search.strip().startswith("search "):                                            # type: ignore
+        if not search.strip().startswith("|"):                                                      
+            if not search.strip().startswith("search "):                                            
                 search = f"search {search}"
 
         # exponential backoff for wait time
@@ -1179,7 +1187,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
 
         return
 
-    def delete_attack_data(self, attack_data_files: list[UnitTestAttackData]):
+    def delete_attack_data(self, attack_data_files: list[TestAttackData]):
         for attack_data_file in attack_data_files:
             index = attack_data_file.custom_index or self.sync_obj.replay_index
             host = attack_data_file.host or self.sync_obj.replay_host
@@ -1212,7 +1220,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
 
     def replay_attack_data_file(
         self,
-        attack_data_file: UnitTestAttackData,
+        attack_data_file: TestAttackData,
         tmp_dir: str,
         test_group: TestGroup,
         test_group_start_time: float,
@@ -1280,7 +1288,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
     def hec_raw_replay(
         self,
         tempfile: str,
-        attack_data_file: UnitTestAttackData,
+        attack_data_file: TestAttackData,
         verify_ssl: bool = False,
     ):
         if verify_ssl is False:

contentctl/actions/detection_testing/views/DetectionTestingView.py

@@ -1,5 +1,6 @@
 import abc
 import datetime
+from typing import Any
 
 from pydantic import BaseModel
 
@@ -10,6 +11,7 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfras
 )
 from contentctl.helper.utils import Utils
 from contentctl.objects.enums import DetectionStatus
+from contentctl.objects.base_test_result import TestResultStatus
 
 
 class DetectionTestingView(BaseModel, abc.ABC):
@@ -74,18 +76,23 @@ class DetectionTestingView(BaseModel, abc.ABC):
         self,
         test_result_fields: list[str] = ["success", "message", "exception", "status", "duration", "wait_duration"],
         test_job_fields: list[str] = ["resultCount", "runDuration"],
-    ) -> dict:
+    ) -> dict[str, dict[str, Any] | list[dict[str, Any]] | str]:
         """
         Iterates over detections, consolidating results into a single dict and aggregating metrics
         :param test_result_fields: fields to pull from the test result
         :param test_job_fields: fields to pull from the job content of the test result
         :returns: summary dict
         """
-        # Init the list of tested detections, and some metrics aggregate counters
-        tested_detections = []
+        # Init the list of tested and skipped detections, and some metrics aggregate counters
+        tested_detections: list[dict[str, Any]] = []
+        skipped_detections: list[dict[str, Any]] = []
         total_pass = 0
         total_fail = 0
         total_skipped = 0
+        total_production = 0
+        total_experimental = 0
+        total_deprecated = 0
+        total_manual = 0
 
         # Iterate the detections tested (anything in the output queue was tested)
         for detection in self.sync_obj.outputQueue:
@@ -95,46 +102,59 @@ class DetectionTestingView(BaseModel, abc.ABC):
             )
 
             # Aggregate detection pass/fail metrics
-            if summary["success"] is False:
+            if detection.test_status == TestResultStatus.FAIL:
                 total_fail += 1
+            elif detection.test_status == TestResultStatus.PASS:
+                total_pass += 1
+            elif detection.test_status == TestResultStatus.SKIP:
+                total_skipped += 1
+
+            # Aggregate production status metrics
+            if detection.status == DetectionStatus.production.value:                                # type: ignore
+                total_production += 1
+            elif detection.status == DetectionStatus.experimental.value:                            # type: ignore
+                total_experimental += 1
+            elif detection.status == DetectionStatus.deprecated.value:                              # type: ignore
+                total_deprecated += 1
+
+            # Check if the detection is manual_test
+            if detection.tags.manual_test is not None:
+                total_manual += 1
+
+            # Append to our list (skipped or tested)
+            if detection.test_status == TestResultStatus.SKIP:
+                skipped_detections.append(summary)
             else:
-                #Test is marked as a success, but we need to determine if there were skipped unit tests
-                #SKIPPED tests still show a success in this field, but we want to count them differently
-                pass_increment = 1
-                for test in summary.get("tests"):
-                    if test.get("test_type") == "unit" and test.get("status") == "skip":
-                        total_skipped += 1
-                        #Test should not count as a pass, so do not increment the count
-                        pass_increment = 0
-                        break
-                total_pass += pass_increment
-                
-
-            # Append to our list
-            tested_detections.append(summary)
-
-        # Sort s.t. all failures appear first (then by name)
-        #Second short condition is a hack to get detections with unit skipped tests to appear above pass tests
-        tested_detections.sort(key=lambda x: (x["success"], 0 if x.get("tests",[{}])[0].get("status","status_missing")=="skip" else 1, x["name"]))
+                tested_detections.append(summary)
 
+        # Sort tested detections s.t. all failures appear first, then by name
+        tested_detections.sort(
+            key=lambda x: (
+                x["success"],
+                x["name"]
+            )
+        )
+
+        # Sort skipped detections s.t. detections w/ tests appear before those w/o, then by name
+        skipped_detections.sort(
+            key=lambda x: (
+                0 if len(x["tests"]) > 0 else 1,
+                x["name"]
+            )
+        )
+
+        # TODO (#267): Align test reporting more closely w/ status enums (as it relates to
+        #   "untested")
         # Aggregate summaries for the untested detections (anything still in the input queue was untested)
         total_untested = len(self.sync_obj.inputQueue)
-        untested_detections = []
+        untested_detections: list[dict[str, Any]] = []
         for detection in self.sync_obj.inputQueue:
             untested_detections.append(detection.get_summary())
 
         # Sort by detection name
         untested_detections.sort(key=lambda x: x["name"])
 
-        # Get lists of detections (name only) that were skipped due to their status (experimental or deprecated)
-        experimental_detections = sorted([
-            detection.name for detection in self.sync_obj.skippedQueue if detection.status == DetectionStatus.experimental.value
-        ])
-        deprecated_detections = sorted([
-            detection.name for detection in self.sync_obj.skippedQueue if detection.status == DetectionStatus.deprecated.value
-        ])
-
-        # If any detection failed, the overall success is False
+        # If any detection failed, or if there are untested detections, the overall success is False
         if (total_fail + len(untested_detections)) == 0:
             overall_success = True
         else:
@@ -143,33 +163,39 @@ class DetectionTestingView(BaseModel, abc.ABC):
         # Compute total detections
         total_detections = total_fail + total_pass + total_untested + total_skipped
 
+        # Compute total detections actually tested (at least one test not skipped)
+        total_tested_detections = total_fail + total_pass
 
         # Compute the percentage of completion for testing, as well as the success rate
         percent_complete = Utils.getPercent(
             len(tested_detections), len(untested_detections), 1
         )
         success_rate = Utils.getPercent(
-            total_pass, total_detections-total_skipped, 1
+            total_pass, total_tested_detections, 1
         )
 
-        # TODO (#230): expand testing metrics reported
+        # TODO (#230): expand testing metrics reported (and make nested)
         # Construct and return the larger results dict
         result_dict = {
             "summary": {
+                "mode": self.config.getModeName(),
+                "enable_integration_testing": self.config.enable_integration_testing,
                 "success": overall_success,
                 "total_detections": total_detections,
+                "total_tested_detections": total_tested_detections,
                 "total_pass": total_pass,
                 "total_fail": total_fail,
                 "total_skipped": total_skipped,
                 "total_untested": total_untested,
-                "total_experimental_or_deprecated": len(deprecated_detections+experimental_detections),
+                "total_production": total_production,
+                "total_experimental": total_experimental,
+                "total_deprecated": total_deprecated,
+                "total_manual": total_manual,
                 "success_rate": success_rate,
             },
             "tested_detections": tested_detections,
+            "skipped_detections": skipped_detections,
             "untested_detections": untested_detections,
             "percent_complete": percent_complete,
-            "deprecated_detections": deprecated_detections,
-            "experimental_detections": experimental_detections
-
         }
         return result_dict

contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py

@@ -45,6 +45,7 @@ class DetectionTestingViewCLI(DetectionTestingView, arbitrary_types_allowed=True
 
         self.showStatus()
 
+    # TODO (#267): Align test reporting more closely w/ status enums (as it relates to "untested")
     def showStatus(self, interval: int = 1):
 
         while True:

contentctl/actions/detection_testing/views/DetectionTestingViewFile.py

@@ -13,7 +13,6 @@ class DetectionTestingViewFile(DetectionTestingView):
     output_filename: str = OUTPUT_FILENAME
 
     def getOutputFilePath(self) -> pathlib.Path:
-        
         folder_path = pathlib.Path('.') / self.output_folder
         output_file = folder_path / self.output_filename
 
@@ -27,13 +26,12 @@ class DetectionTestingViewFile(DetectionTestingView):
         output_file = self.getOutputFilePath()
 
         folder_path.mkdir(parents=True, exist_ok=True)
-        
-        
+
         result_dict = self.getSummaryObject()
-        
+
         # use the yaml writer class
         with open(output_file, "w") as res:
-            res.write(yaml.safe_dump(result_dict,sort_keys=False))
+            res.write(yaml.safe_dump(result_dict, sort_keys=False))
 
     def showStatus(self, interval: int = 60):
         pass

contentctl/actions/test.py

@@ -44,35 +44,25 @@ class TestInputDto:
     
 
 class Test:
+    def filter_tests(self, input_dto: TestInputDto) -> None:
+        """
+        If integration testing has NOT been enabled, then skip
+        all of the integration tests. Otherwise, do nothing
+
+        Args:
+            input_dto (TestInputDto): A configuration of the test and all of the
+            tests to be run.
+        """        
 
-    def filter_detections(self, input_dto: TestInputDto)->TestInputDto:
-        
         if not input_dto.config.enable_integration_testing:
-            #Skip all integraiton tests if integration testing is not enabled:
+            # Skip all integraiton tests if integration testing is not enabled:
             for detection in input_dto.detections:
                 for test in detection.tests:
                     if isinstance(test, IntegrationTest):
                         test.skip("TEST SKIPPED: Skipping all integration tests")
-        
-        list_after_filtering:List[Detection] = []
-        #extra filtering which may be removed/modified in the future
-        for detection in input_dto.detections:
-            if (detection.status != DetectionStatus.production.value):
-                #print(f"{detection.name} - Not testing because [STATUS: {detection.status}]")
-                pass
-            elif detection.type == AnalyticsType.Correlation:
-                #print(f"{detection.name} - Not testing because [  TYPE: {detection.type}]")
-                pass
-            else:
-                list_after_filtering.append(detection)
-        
-        return TestInputDto(list_after_filtering, input_dto.config)
-        
-        
-    def execute(self, input_dto: TestInputDto) -> bool:
 
         
-
+    def execute(self, input_dto: TestInputDto) -> bool:
         output_dto = DetectionTestingManagerOutputDto()
 
         web = DetectionTestingViewWeb(config=input_dto.config, sync_obj=output_dto)
@@ -87,26 +77,33 @@ class Test:
         manager = DetectionTestingManager(
             input_dto=manager_input_dto, output_dto=output_dto
         )
-        
+
+        mode = input_dto.config.getModeName()
         if len(input_dto.detections) == 0:
-            print(f"With Detection Testing Mode '{input_dto.config.getModeName()}', there were [0] detections found to test.\nAs such, we will quit immediately.")
-            # Directly call stop so that the summary.yml will be generated. Of course it will not have any test results, but we still want it to contain
-            # a summary showing that now detections were tested.
+            print(
+                f"With Detection Testing Mode '{mode}', there were [0] detections found to test."
+                "\nAs such, we will quit immediately."
+            )
+            # Directly call stop so that the summary.yml will be generated. Of course it will not
+            # have any test results, but we still want it to contain a summary showing that now
+            # detections were tested.
             file.stop()
         else:
-            print(f"MODE: [{input_dto.config.getModeName()}] - Test [{len(input_dto.detections)}] detections")
-            if input_dto.config.mode in [DetectionTestingMode.changes, DetectionTestingMode.selected]:
-                files_string = '\n- '.join([str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections])
+            print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections")
+            if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]:
+                files_string = '\n- '.join(
+                    [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections]
+                )
                 print(f"Detections:\n- {files_string}")
 
             manager.setup()
             manager.execute()
-        
+
         try:
             summary_results = file.getSummaryObject()
             summary = summary_results.get("summary", {})
 
-            print("Test Summary")
+            print(f"Test Summary (mode: {summary.get('mode','Error')})")
             print(f"\tSuccess                      : {summary.get('success',False)}")
             print(
                 f"\tSuccess Rate                 : {summary.get('success_rate','ERROR')}"
@@ -115,15 +112,41 @@ class Test:
                 f"\tTotal Detections             : {summary.get('total_detections','ERROR')}"
             )
             print(
-                f"\tPassed Detections            : {summary.get('total_pass','ERROR')}"
+                f"\tTotal Tested Detections      : {summary.get('total_tested_detections','ERROR')}"
             )
             print(
-                f"\tFailed Detections            : {summary.get('total_fail','ERROR')}"
+                f"\t  Passed Detections          : {summary.get('total_pass','ERROR')}"
+            )
+            print(
+                f"\t  Failed Detections          : {summary.get('total_fail','ERROR')}"
+            )
+            print(
+                f"\tSkipped Detections           : {summary.get('total_skipped','ERROR')}"
+            )
+            print(
+                "\tProduction Status            :"
+            )
+            print(
+                f"\t  Production Detections      : {summary.get('total_production','ERROR')}"
+            )
+            print(
+                f"\t  Experimental Detections    : {summary.get('total_experimental','ERROR')}"
+            )
+            print(
+                f"\t  Deprecated Detections      : {summary.get('total_deprecated','ERROR')}"
+            )
+            print(
+                f"\tManually Tested Detections : {summary.get('total_manual','ERROR')}"
             )
             print(
                 f"\tUntested Detections          : {summary.get('total_untested','ERROR')}"
             )
             print(f"\tTest Results File            : {file.getOutputFilePath()}")
+            print(
+                "\nNOTE: skipped detections include non-production, manually tested, and certain\n"
+                "detection types (e.g. Correlation), but there may be overlap between these\n"
+                "categories."
+            )
             return summary_results.get("summary", {}).get("success", False)
 
         except Exception as e:

contentctl/contentctl.py

@@ -113,17 +113,14 @@ def test_common_func(config:test_common):
     test_input_dto = TestInputDto(detections_to_test, config)
     
     t = Test()
-    
-    # Remove detections that we do not want to test because they are
-    # not production, the correct type, or manual_test only
-    filted_test_input_dto = t.filter_detections(test_input_dto)
+    t.filter_tests(test_input_dto)
     
     if config.plan_only:
         #Emit the test plan and quit. Do not actually run the test
-        config.dumpCICDPlanAndQuit(gitServer.getHash(),filted_test_input_dto.detections)
+        config.dumpCICDPlanAndQuit(gitServer.getHash(),test_input_dto.detections)
         return 
     
-    success = t.execute(filted_test_input_dto)
+    success = t.execute(test_input_dto)
     
     if success:
         #Everything passed!

contentctl/enrichments/attack_enrichment.py

@@ -10,6 +10,7 @@ from dataclasses import field
 from typing import Annotated,Any
 from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
 from contentctl.objects.config import validate
+from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE
 logging.getLogger('taxii2client').setLevel(logging.CRITICAL)
 
 
@@ -23,7 +24,7 @@ class AttackEnrichment(BaseModel):
         _ = enrichment.get_attack_lookup(str(config.path))
         return enrichment
     
-    def getEnrichmentByMitreID(self, mitre_id:Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})?$")])->MitreAttackEnrichment:
+    def getEnrichmentByMitreID(self, mitre_id:MITRE_ATTACK_ID_TYPE)->MitreAttackEnrichment:
         if not self.use_enrichment:
             raise Exception(f"Error, trying to add Mitre Enrichment, but use_enrichment was set to False")
         

contentctl/enrichments/cve_enrichment.py

@@ -8,7 +8,7 @@ from typing import Annotated, Any, Union, TYPE_CHECKING
 from pydantic import BaseModel,Field, computed_field
 from decimal import Decimal
 from requests.exceptions import ReadTimeout
-
+from contentctl.objects.annotated_types import CVE_TYPE
 if TYPE_CHECKING:
     from contentctl.objects.config import validate
 
@@ -18,7 +18,7 @@ CVESSEARCH_API_URL = 'https://cve.circl.lu'
 
 
 class CveEnrichmentObj(BaseModel):
-    id: Annotated[str, r"^CVE-[1|2]\d{3}-\d+$"]
+    id: CVE_TYPE
     cvss: Annotated[Decimal, Field(ge=.1, le=10, decimal_places=1)]
     summary: str