contentctl 5.1.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

contentctl/actions/test.py

@@ -1,44 +1,36 @@
+import pathlib
 from dataclasses import dataclass
 from typing import List
 
-from contentctl.objects.config import test_common, Selected, Changes
-from contentctl.objects.detection import Detection
-
-
 from contentctl.actions.detection_testing.DetectionTestingManager import (
     DetectionTestingManager,
     DetectionTestingManagerInputDto,
 )
-
-
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
     DetectionTestingManagerOutputDto,
 )
-
-
-from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import (
-    DetectionTestingViewWeb,
-)
-
 from contentctl.actions.detection_testing.views.DetectionTestingViewCLI import (
     DetectionTestingViewCLI,
 )
-
 from contentctl.actions.detection_testing.views.DetectionTestingViewFile import (
     DetectionTestingViewFile,
 )
-
+from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import (
+    DetectionTestingViewWeb,
+)
+from contentctl.objects.config import Changes, Selected
+from contentctl.objects.config import test as test_
+from contentctl.objects.config import test_servers
+from contentctl.objects.detection import Detection
 from contentctl.objects.integration_test import IntegrationTest
 
-import pathlib
-
 MAXIMUM_CONFIGURATION_TIME_SECONDS = 600
 
 
 @dataclass(frozen=True)
 class TestInputDto:
     detections: List[Detection]
-    config: test_common
+    config: test_ | test_servers
 
 
 class Test:
@@ -77,8 +69,8 @@ class Test:
 
         if len(input_dto.detections) == 0:
             print(
-                f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were [0] detections found to test."
-                "\nAs such, we will quit immediately."
+                f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were "
+                "[0] detections found to test.\nAs such, we will quit immediately."
             )
             # Directly call stop so that the summary.yml will be generated. Of course it will not
             # have any test results, but we still want it to contain a summary showing that now
@@ -109,6 +101,10 @@ class Test:
         try:
             summary_results = file.getSummaryObject()
             summary = summary_results.get("summary", {})
+            if not isinstance(summary, dict):
+                raise ValueError(
+                    f"Summary in results was an unexpected type ({type(summary)}): {summary}"
+                )
 
             print(f"Test Summary (mode: {summary.get('mode', 'Error')})")
             print(f"\tSuccess                      : {summary.get('success', False)}")
@@ -152,7 +148,7 @@ class Test:
                 "detection types (e.g. Correlation), but there may be overlap between these\n"
                 "categories."
             )
-            return summary_results.get("summary", {}).get("success", False)
+            return summary.get("success", False)
 
         except Exception as e:
             print(f"Error determining if whole test was successful: {str(e)}")

contentctl/actions/validate.py

@@ -1,14 +1,14 @@
 import pathlib
 
-from contentctl.input.director import Director, DirectorOutputDto
-from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
-from contentctl.objects.atomic import AtomicEnrichment
-from contentctl.objects.lookup import FileBackedLookup
+from contentctl.helper.splunk_app import SplunkApp
 from contentctl.helper.utils import Utils
+from contentctl.input.director import Director, DirectorOutputDto
+from contentctl.objects.atomic import AtomicEnrichment
+from contentctl.objects.config import validate
 from contentctl.objects.data_source import DataSource
-from contentctl.helper.splunk_app import SplunkApp
+from contentctl.objects.lookup import FileBackedLookup, RuntimeCSV
 
 
 class Validate:
@@ -17,16 +17,6 @@ class Validate:
             AtomicEnrichment.getAtomicEnrichment(input_dto),
             AttackEnrichment.getAttackEnrichment(input_dto),
             CveEnrichment.getCveEnrichment(input_dto),
-            [],
-            [],
-            [],
-            [],
-            [],
-            [],
-            [],
-            [],
-            [],
-            [],
         )
 
         director = Director(director_output_dto)
@@ -68,7 +58,10 @@ class Validate:
         usedLookupFiles: list[pathlib.Path] = [
             lookup.filename
             for lookup in director_output_dto.lookups
+            # Of course Runtime CSVs do not have underlying CSV files, so make
+            # sure that we do not check for that existence.
             if isinstance(lookup, FileBackedLookup)
+            and not isinstance(lookup, RuntimeCSV)
         ] + [
             lookup.file_path
             for lookup in director_output_dto.lookups

contentctl/helper/utils.py

@@ -1,40 +1,34 @@
-import os
-import git
-import shutil
-import requests
+import logging
+import pathlib
 import random
+import shutil
 import string
+from math import ceil
 from timeit import default_timer
-import pathlib
+from typing import TYPE_CHECKING, Tuple, Union
 
-from typing import Union, Tuple
+import git
+import requests
 import tqdm
-from math import ceil
-
-from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.security_content_object import SecurityContentObject
 
-
 TOTAL_BYTES = 0
 ALWAYS_PULL = True
 
 
 class Utils:
     @staticmethod
-    def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]:
-        listOfFiles: list[pathlib.Path] = []
-        base_path = pathlib.Path(path)
-        if not base_path.exists():
-            return listOfFiles
-        for dirpath, dirnames, filenames in os.walk(path):
-            for file in filenames:
-                if file.endswith(".yml"):
-                    listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
+    def get_all_yml_files_from_directory(path: pathlib.Path) -> list[pathlib.Path]:
+        if not path.exists():
+            raise FileNotFoundError(
+                f"Trying to find files in the directory '{path.absolute()}', but it does not exist.\n"
+                "It is not mandatory to have content/YMLs in this directory, but it must exist. Please create it."
+            )
 
-        return sorted(listOfFiles)
+        return sorted(pathlib.Path(yml_path) for yml_path in path.glob("**/*.yml"))
 
     @staticmethod
     def get_security_content_files_from_directory(
@@ -490,3 +484,58 @@ class Utils:
         ratio = numerator / denominator
         percent = ratio * 100
         return Utils.getFixedWidth(percent, decimal_places) + "%"
+
+    @staticmethod
+    def get_logger(
+        name: str, log_level: int, log_path: str, enable_logging: bool
+    ) -> logging.Logger:
+        """
+        Gets a logger instance for the given name; logger is configured if not already configured.
+        The NullHandler is used to suppress loggging when running in production so as not to
+        conflict w/ contentctl's larger pbar-based logging. The StreamHandler is enabled by setting
+        enable_logging to True (useful for debugging/testing locally)
+
+        :param name: the logger name
+        :type name: str
+        :param log_level: the logging level (e.g. `logging.Debug`)
+        :type log_level: int
+        :param log_path: the path for the log file
+        :type log_path: str
+        :param enable_logging: a flag indicating whether logging should be redirected from null to
+            the stream handler
+        :type enable_logging: bool
+
+        :return: a logger
+        :rtype: :class:`logging.Logger`
+        """
+        # get logger for module
+        logger = logging.getLogger(name)
+
+        # set propagate to False if not already set as such (needed to that we do not flow up to any
+        # root loggers)
+        if logger.propagate:
+            logger.propagate = False
+
+        # if logger has no handlers, it needs to be configured for the first time
+        if not logger.hasHandlers():
+            # set level
+            logger.setLevel(log_level)
+
+            # if logging enabled, use a StreamHandler; else, use the NullHandler to suppress logging
+            handler: logging.Handler
+            if enable_logging:
+                handler = logging.FileHandler(log_path)
+            else:
+                handler = logging.NullHandler()
+
+            # Format our output
+            formatter = logging.Formatter(
+                "%(asctime)s - %(levelname)s:%(name)s - %(message)s"
+            )
+            handler.setFormatter(formatter)
+
+            # Set handler level and add to logger
+            handler.setLevel(log_level)
+            logger.addHandler(handler)
+
+        return logger

contentctl/input/director.py

@@ -1,29 +1,42 @@
-import os
 import sys
 from dataclasses import dataclass, field
 from pathlib import Path
 from uuid import UUID
 
-from pydantic import ValidationError
+from pydantic import TypeAdapter, ValidationError
 
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.helper.utils import Utils
 from contentctl.input.yml_reader import YmlReader
+from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
+    DeprecationDocumentationFile,
+)
 from contentctl.objects.atomic import AtomicEnrichment
 from contentctl.objects.baseline import Baseline
-from contentctl.objects.config import validate
+from contentctl.objects.config import CustomApp, validate
 from contentctl.objects.dashboard import Dashboard
 from contentctl.objects.data_source import DataSource
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.detection import Detection
-from contentctl.objects.enums import SecurityContentType
 from contentctl.objects.investigation import Investigation
-from contentctl.objects.lookup import Lookup, LookupAdapter
+from contentctl.objects.lookup import (
+    CSVLookup,
+    KVStoreLookup,
+    Lookup,
+    Lookup_Type,
+    LookupAdapter,
+    MlModel,
+    RuntimeCSV,
+)
 from contentctl.objects.macro import Macro
 from contentctl.objects.playbook import Playbook
+from contentctl.objects.removed_security_content_object import (
+    RemovedSecurityContentObject,
+)
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.story import Story
+from contentctl.output.runtime_csv_writer import RuntimeCsvWriter
 
 
 @dataclass
@@ -33,17 +46,20 @@ class DirectorOutputDto:
     atomic_enrichment: AtomicEnrichment
     attack_enrichment: AttackEnrichment
     cve_enrichment: CveEnrichment
-    detections: list[Detection]
-    stories: list[Story]
-    baselines: list[Baseline]
-    investigations: list[Investigation]
-    playbooks: list[Playbook]
-    macros: list[Macro]
-    lookups: list[Lookup]
-    deployments: list[Deployment]
-    dashboards: list[Dashboard]
-
-    data_sources: list[DataSource]
+    detections: list[Detection] = field(default_factory=list)
+    stories: list[Story] = field(default_factory=list)
+    baselines: list[Baseline] = field(default_factory=list)
+    investigations: list[Investigation] = field(default_factory=list)
+    playbooks: list[Playbook] = field(default_factory=list)
+    macros: list[Macro] = field(default_factory=list)
+    lookups: list[Lookup] = field(default_factory=list)
+    deployments: list[Deployment] = field(default_factory=list)
+    dashboards: list[Dashboard] = field(default_factory=list)
+    deprecated: list[RemovedSecurityContentObject] = field(default_factory=list)
+    data_sources: list[DataSource] = field(default_factory=list)
+    deprecation_documentation: DeprecationDocumentationFile = field(
+        default_factory=DeprecationDocumentationFile
+    )
     name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
     uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
 
@@ -82,9 +98,10 @@ class DirectorOutputDto:
             self.detections.append(content)
         elif isinstance(content, Dashboard):
             self.dashboards.append(content)
-
         elif isinstance(content, DataSource):
             self.data_sources.append(content)
+        elif isinstance(content, RemovedSecurityContentObject):
+            self.deprecated.append(content)
         else:
             raise Exception(f"Unknown security content type: {type(content)}")
 
@@ -101,123 +118,134 @@ class Director:
 
     def execute(self, input_dto: validate) -> None:
         self.input_dto = input_dto
-        self.createSecurityContent(SecurityContentType.deployments)
-        self.createSecurityContent(SecurityContentType.lookups)
-        self.createSecurityContent(SecurityContentType.macros)
-        self.createSecurityContent(SecurityContentType.stories)
-        self.createSecurityContent(SecurityContentType.baselines)
-        self.createSecurityContent(SecurityContentType.investigations)
-        self.createSecurityContent(SecurityContentType.data_sources)
-        self.createSecurityContent(SecurityContentType.playbooks)
-        self.createSecurityContent(SecurityContentType.detections)
-        self.createSecurityContent(SecurityContentType.dashboards)
-
-    def createSecurityContent(self, contentType: SecurityContentType) -> None:
-        if contentType in [
-            SecurityContentType.deployments,
-            SecurityContentType.lookups,
-            SecurityContentType.macros,
-            SecurityContentType.stories,
-            SecurityContentType.baselines,
-            SecurityContentType.investigations,
-            SecurityContentType.playbooks,
-            SecurityContentType.detections,
-            SecurityContentType.data_sources,
-            SecurityContentType.dashboards,
+
+        for content in [
+            Deployment,
+            LookupAdapter,
+            Macro,
+            Story,
+            Baseline,
+            DataSource,
+            Playbook,
+            Detection,
+            Dashboard,
+            RemovedSecurityContentObject,
         ]:
-            files = Utils.get_all_yml_files_from_directory(
-                os.path.join(self.input_dto.path, str(contentType.name))
-            )
-            security_content_files = [f for f in files]
-        else:
-            raise (
-                Exception(
-                    f"Cannot createSecurityContent for unknown product {contentType}."
+            self.createSecurityContent(content)
+
+        self.loadDeprecationInfo(input_dto.app)
+        self.buildRuntimeCsvs()
+
+    def buildRuntimeCsvs(self):
+        self.buildDataSourceCsv()
+        self.buildDeprecationRemovalCsv()
+
+    def buildDeprecationRemovalCsv(self):
+        if self.input_dto.enforce_deprecation_mapping_requirement is False:
+            # Do not build the CSV, it would be wasteful to include it if it
+            # is not even used
+            return
+        deprecation_lookup = RuntimeCSV(
+            name="deprecation_info",
+            id=UUID("99262bf2-9606-4b52-b377-c96713527b35"),
+            version=1,
+            author=self.input_dto.app.author_name,
+            description="A lookup file that contains information about content that has been deprecated or removed from the app.",
+            lookup_type=Lookup_Type.csv,
+            contents=RuntimeCsvWriter.generateDeprecationCSVContent(
+                self.output_dto, self.input_dto.app
+            ),
+        )
+        self.output_dto.addContentToDictMappings(deprecation_lookup)
+
+    def buildDataSourceCsv(self):
+        datasource_lookup = RuntimeCSV(
+            name="data_sources",
+            id=UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
+            version=1,
+            author=self.input_dto.app.author_name,
+            description="A lookup file that contains the data source objects for detections.",
+            lookup_type=Lookup_Type.csv,
+            contents=RuntimeCsvWriter.generateDatasourceCSVContent(
+                self.output_dto.data_sources
+            ),
+        )
+        self.output_dto.addContentToDictMappings(datasource_lookup)
+
+    def loadDeprecationInfo(self, app: CustomApp):
+        mapping_file_paths = list(
+            (self.input_dto.path / "removed").glob("deprecation_mapping*.YML")
+        )
+
+        if self.input_dto.enforce_deprecation_mapping_requirement is False:
+            # If we are not required to enforce deprecation mapping, then do nothing at all (even if the files exist)
+            if len(mapping_file_paths) > 0:
+                file_paths = "\n - " + "\n - ".join(
+                    str(name) for name in mapping_file_paths
                 )
-            )
+                print(
+                    "The following deprecation_mapping*.YML files were found, but will not be parsed because "
+                    f"[enforce_deprecation_mapping_requirement = {self.input_dto.enforce_deprecation_mapping_requirement}]:",
+                    file_paths,
+                )
+            # Otherwise, no need to output extra information
+            return
+
+        # If there are no mapping files, that's okay.  We will other throw exceptions later on if
+        # there are 1 or more detections marked as deprecated or removed.
+        for mapping_file_path in mapping_file_paths:
+            print(f"Parsing mapping file {mapping_file_path.name}")
+            data = YmlReader.load_file(mapping_file_path)
+            mapping = DeprecationDocumentationFile.model_validate(data)
+            self.output_dto.deprecation_documentation += mapping
+
+        self.output_dto.deprecation_documentation.mapAllContent(self.output_dto, app)
+
+    def createSecurityContent(
+        self,
+        contentType: type[SecurityContentObject]
+        | TypeAdapter[CSVLookup | KVStoreLookup | MlModel],
+    ) -> None:
+        files = Utils.get_all_yml_files_from_directory(
+            self.input_dto.path / contentType.containing_folder()  # type: ignore
+        )
+
+        # convert this generator to a list so that we can
+        # calculate progress as we iterate over the files
+        security_content_files = [f for f in files]
 
         validation_errors: list[tuple[Path, ValueError]] = []
 
         already_ran = False
         progress_percent = 0
+        context: dict[str, validate | DirectorOutputDto] = {
+            "output_dto": self.output_dto,
+            "config": self.input_dto,
+        }
+        contentCartegoryName: str = contentType.__name__.upper()  # type: ignore
 
         for index, file in enumerate(security_content_files):
             progress_percent = ((index + 1) / len(security_content_files)) * 100
             try:
-                type_string = contentType.name.upper()
+                type_string = contentType.__name__.upper()  # type: ignore
                 modelDict = YmlReader.load_file(file)
 
-                if contentType == SecurityContentType.lookups:
-                    lookup = LookupAdapter.validate_python(
-                        modelDict,
-                        context={
-                            "output_dto": self.output_dto,
-                            "config": self.input_dto,
-                        },
-                    )
-                    # lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
-                    self.output_dto.addContentToDictMappings(lookup)
-
-                elif contentType == SecurityContentType.macros:
-                    macro = Macro.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
+                if isinstance(contentType, type(SecurityContentObject)):
+                    content: SecurityContentObject = contentType.model_validate(
+                        modelDict, context=context
                     )
-                    self.output_dto.addContentToDictMappings(macro)
-
-                elif contentType == SecurityContentType.deployments:
-                    deployment = Deployment.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
+                elif contentType == LookupAdapter:
+                    content: SecurityContentObject = (  # type: ignore
+                        contentType.validate_python(modelDict, context=context)  # type:ignore
                     )
-                    self.output_dto.addContentToDictMappings(deployment)
-
-                elif contentType == SecurityContentType.playbooks:
-                    playbook = Playbook.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(playbook)
-
-                elif contentType == SecurityContentType.baselines:
-                    baseline = Baseline.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(baseline)
-
-                elif contentType == SecurityContentType.investigations:
-                    investigation = Investigation.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(investigation)
-
-                elif contentType == SecurityContentType.stories:
-                    story = Story.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(story)
-
-                elif contentType == SecurityContentType.detections:
-                    detection = Detection.model_validate(
-                        modelDict,
-                        context={
-                            "output_dto": self.output_dto,
-                            "app": self.input_dto.app,
-                        },
-                    )
-                    self.output_dto.addContentToDictMappings(detection)
-
-                elif contentType == SecurityContentType.dashboards:
-                    dashboard = Dashboard.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(dashboard)
-
-                elif contentType == SecurityContentType.data_sources:
-                    data_source = DataSource.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.addContentToDictMappings(data_source)
-
+                    if not isinstance(content, SecurityContentObject):
+                        raise Exception(
+                            f"Expected lookup to be a SecurityContentObject (CSVLookup, KVStoreLookup, or MLModel), but it was actually: {type(content)}"  # type: ignore
+                        )
                 else:
-                    raise Exception(f"Unsupported type: [{contentType}]")
+                    raise Exception(f"Unknown contentType in Director: {contentType}")
+
+                self.output_dto.addContentToDictMappings(content)
 
                 if (
                     sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()
@@ -236,7 +264,7 @@ class Director:
                 validation_errors.append((relative_path, e))
 
         print(
-            f"\r{f'{contentType.name.upper()} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...",
+            f"\r{f'{contentCartegoryName} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...",
             end="",
             flush=True,
         )

contentctl/input/yml_reader.py

@@ -1,7 +1,8 @@
-from typing import Dict, Any
-import yaml
-import sys
 import pathlib
+import sys
+from typing import Any, Dict
+
+import yaml
 
 
 class YmlReader:
@@ -13,40 +14,51 @@ class YmlReader:
     ) -> Dict[str, Any]:
         try:
             file_handler = open(file_path, "r", encoding="utf-8")
+        except OSError as exc:
+            print(
+                f"\nThere was an unrecoverable error when opening the file '{file_path}' - we will exit immediately:\n{str(exc)}"
+            )
+            sys.exit(1)
 
             # The following code can help diagnose issues with duplicate keys or
             # poorly-formatted but still "compliant" YML.  This code should be
             # enabled manually for debugging purposes. As such, strictyaml
             # library is intentionally excluded from the contentctl requirements
 
+        try:
             if STRICT_YML_CHECKING:
+                # This is an extra level of verbose parsing that can be
+                # enabled for debugging purpose. It is intentionally done in
+                # addition to the regular yml parsing
                 import strictyaml
 
-                try:
-                    strictyaml.dirty_load(file_handler.read(), allow_flow_style=True)
-                    file_handler.seek(0)
-                except Exception as e:
-                    print(f"Error loading YML file {file_path}: {str(e)}")
-                    sys.exit(1)
-            try:
-                # Ideally we should use
-                # from contentctl.actions.new_content import NewContent
-                # and use NewContent.UPDATE_PREFIX,
-                # but there is a circular dependency right now which makes that difficult.
-                # We have instead hardcoded UPDATE_PREFIX
-                UPDATE_PREFIX = "__UPDATE__"
-                data = file_handler.read()
-                if UPDATE_PREFIX in data:
-                    raise Exception(
-                        f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required."
-                    )
-                yml_obj = yaml.load(data, Loader=yaml.CSafeLoader)
-            except yaml.YAMLError as exc:
-                print(exc)
-                sys.exit(1)
+                strictyaml.dirty_load(file_handler.read(), allow_flow_style=True)
+                file_handler.seek(0)
 
-        except OSError as exc:
-            print(exc)
+            # Ideally we should use
+            # from contentctl.actions.new_content import NewContent
+            # and use NewContent.UPDATE_PREFIX,
+            # but there is a circular dependency right now which makes that difficult.
+            # We have instead hardcoded UPDATE_PREFIX
+            UPDATE_PREFIX = "__UPDATE__"
+            data = file_handler.read()
+            if UPDATE_PREFIX in data:
+                raise Exception(
+                    f"\nThe file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required."
+                )
+            yml_obj = yaml.load(data, Loader=yaml.CSafeLoader)
+            if yml_obj is None:
+                raise yaml.YAMLError(
+                    f"The YML file's value was parsed as [{None}]. "
+                    "This probably means that the file was entirely "
+                    "empty or contains only comments, which is not "
+                    "supported. Please ensure this file is NOT empty "
+                    "or remove the file."
+                )
+        except yaml.YAMLError as exc:
+            print(
+                f"\nThere was an unrecoverable YML Parsing error when reading or parsing the file '{file_path}' - we will exit immediately:\n{str(exc)}"
+            )
             sys.exit(1)
 
         if add_fields is False: