contentctl-ng 0.1.0__py3-none-any.whl

contentctl_ng/__init__.py

@@ -0,0 +1,10 @@
+"""Allows contentctl_ng to be used as a package."""
+
+import os
+import sys
+
+TQDM_DISABLE_ENV_VAR = "TQDM_DISABLE"
+
+if not sys.stderr.isatty() and TQDM_DISABLE_ENV_VAR not in os.environ:
+    # If stderr is not a terminal and the envvar is not already set, disable tqdm progress bars to avoid messy output
+    os.environ[TQDM_DISABLE_ENV_VAR] = "true"

contentctl_ng/__main__.py

@@ -0,0 +1,10 @@
+"""Allows the main cli entry to be invoked when running contentctl_ng as a module.
+
+i.e.:
+python -m contentctl_ng install
+"""
+
+from contentctl_ng.cli import main
+
+if __name__ == "__main__":
+    main()

contentctl_ng/build.py

@@ -0,0 +1,635 @@
+"""Very simple demo that will parse security content objects."""
+
+import logging
+import shutil
+import tarfile
+from csv import QUOTE_ALL, DictWriter
+from pathlib import Path
+from typing import Any, Literal
+from uuid import UUID
+
+from pydantic_settings import SettingsConfigDict
+
+import contentctl_ng.utilities.reporting_tools as reporting_tools
+from contentctl_ng.objects.conf import (
+    AppDotConf,
+    ConfFile,
+    ContentVersionDotConf,
+    DistSearchDotConf,
+    ServerDotConf,
+)
+from contentctl_ng.objects.content import DeprecatableSecurityContent, SecurityContent
+from contentctl_ng.objects.lookup import CSVLookup
+from contentctl_ng.objects.object_parser import AppFactory
+from contentctl_ng.settings import ContentSettings
+
+logger = logging.getLogger(__name__)
+
+
+def write_app_template(app_factory: AppFactory, built_app_directory: Path) -> None:
+    """Write everything in the app template to the build directory.
+
+    This will combine the app_templates from all of the directories/repos
+    that make up the app. Any collisions/attempts to overwrite
+    files in app_template directories will result in an exception.
+    For instance, if directory 1 contains:
+    security_content_github/app_template/default.meta
+    AND directory 2 contains:
+    security_content_gitlab/app_template/default.meta
+    then an exception will be raised.
+
+    Args:
+        app_factory: The app factory to use to generate the app template.
+        built_app_directory: The directory to write the app template to.
+
+    Returns:
+        None - but as a result the app template is written to the built_app_directory
+    """
+    for directory in app_factory.app._content_directories:  # pyright: ignore[reportPrivateUsage]
+        app_template_directory = directory.path / "app_template"
+
+        for templated_file in app_template_directory.rglob("*"):
+            if not templated_file.is_file():
+                # Only worry about copying files - do not empty create directory structure.
+                continue
+
+            # Get the file relative to the path of the new app
+            relative_path = templated_file.relative_to(app_template_directory)
+            build_file_path = built_app_directory / relative_path
+            if build_file_path.exists():
+                raise Exception(
+                    f"Error writing app template from directory "
+                    f"{app_template_directory}: File {build_file_path} already exists."
+                    "Please remove it before running this script."
+                )
+
+            # The file does not exist, so copy it.
+            # We must make sure that the target folder exists first
+            build_file_path.parent.mkdir(exist_ok=True, parents=True)
+            templated_file.copy(build_file_path)
+
+    return None
+
+
+def write_conf_files(app_factory: AppFactory, built_app_directory: Path) -> None:
+    """Generate all of the conf files required for the app.
+
+    Args:
+        app_factory: The app factory to use to generate the conf files.
+        built_app_directory: The directory to write the conf files to.
+
+    Returns:
+        None - but as a result the app is built in the built_app_directory
+    """
+    conf_directory = built_app_directory / "default"
+    # It's okay if this exists becuase it may have been created
+    # via the app_template.  However, it's also okay if
+    # it DOESN'T exist.
+    conf_directory.mkdir(exist_ok=True, parents=True)
+
+    # These confs files do not require any special actions or iteration so just emit them.
+    in_memory_conf_files: dict[str, ConfFile] = {}
+
+    # Construct all the file object in memory
+    # This is done ONCE, not once per repo directory for the app
+    # We constuct these files in memory first so that we can later
+    # serialize them all at once.
+
+    # The following can be constructed without SecurityContent
+    for conf_object in (
+        AppDotConf,
+        DistSearchDotConf,
+        ContentVersionDotConf,
+        ServerDotConf,
+    ):
+        conf = conf_object(
+            directory_path=conf_directory,
+            app_settings=app_factory.app._shared_app_settings,  # pyright: ignore[reportPrivateUsage]
+        )
+        in_memory_conf_files[conf.file_name] = conf
+
+    # Construct a list so that we do not need to duplicate the serialization code.
+    # First we have the filename. Then, we have a list of content along with the name
+    # of its serialization method. For instance the following object is annotated
+    # for informational purposes:
+    # Each tuple in the list is in the following format:
+    # tuple[0]: str - the name of the conf file to serialize to
+    # tuple[1]: list[list[object, str]] - a list of objects and their serialization methods
+    # tuple[1][0]: list[object, str] - a list of objects and their serialization methods
+
+    conf_file_to_content_mapping: list[tuple[str, list[tuple[Any, str]]]] = [
+        ("macros.conf", [(app_factory.macros, "emit_ConfStanza")]),
+        (
+            "transforms.conf",
+            [
+                (
+                    app_factory.csv_lookups + app_factory.kvstore_lookups,
+                    "emit_ConfStanza",
+                )
+            ],
+        ),
+        # analyticstories.conf contains two different types of content:
+        # stories and detections in different formats.
+        # We need to serialize them differently.
+        (
+            "analyticstories.conf",
+            [
+                (app_factory.stories, "emit_ConfStanza"),
+                (app_factory.detections, "emit_ConfStanza_analyticstories"),
+            ],
+        ),
+        (
+            "collections.conf",
+            [(app_factory.kvstore_lookups, "emit_ConfStanza_collections")],
+        ),
+        # savedsearches contains two different types of content:
+        # detections and baselines in different formats.
+        # We need to serialize them differently.
+        (
+            "savedsearches.conf",
+            [
+                (app_factory.detections, "emit_ConfStanza_savedsearches"),
+                (app_factory.baselines, "emit_ConfStanza_savedsearches"),
+            ],
+        ),
+    ]
+
+    # Iterate over all the
+    for conf_file_name, conf_file_contents in conf_file_to_content_mapping:
+        conf = ConfFile(
+            file_name=conf_file_name,
+            directory_path=conf_directory,
+            app_settings=app_factory.app._shared_app_settings,  # pyright: ignore[reportPrivateUsage]
+        )
+        if in_memory_conf_files.get(conf.file_name, None) is not None:
+            raise Exception(
+                f"Conf file {conf.file_name} already exists in the in_memory_conf_files dictionary. It must not be recreated."
+            )
+
+        in_memory_conf_files[conf.file_name] = conf
+
+        # Iterate over all the types of content that must go into a given conf file
+        for content_object, serialization_method_name in conf_file_contents:
+            # Iterate over all the objects of a given type
+            for obj in content_object:
+                # Add this object's stanza to the conf file.
+                conf.add_stanza(getattr(obj, serialization_method_name)())
+
+    # write out all the file objects. For the same of consistency, we sort them by file name.
+    for conf_file in sorted(in_memory_conf_files.values(), key=lambda x: x.file_name):
+        conf_file.serialize_to_file()
+
+    # NOTE: this will be removed once FBDs have proper support in contentctl-ng
+    # After writing savedsearches.conf, append static FBD content if it exists
+    append_savedsearches_fbd_conf(app_factory, conf_directory)
+
+    return None
+
+
+# NOTE: this will be removed once FBDs have proper support in contentctl-ng
+def parse_savedsearches_fbd_conf(fbd_conf_file: Path) -> list[str]:
+    """Parse savedsearches_fbd.conf file and extract stanzas, skipping comments and blank lines.
+
+    Args:
+        fbd_conf_file: Path to the savedsearches_fbd.conf file
+
+    Returns:
+        List of stanza strings (each string is a complete stanza)
+    """
+    stanzas: list[str] = []
+    current_stanza_lines: list[str] = []
+    has_stanza_header = False
+
+    with open(fbd_conf_file, "r") as f:
+        for line in f:
+            stripped = line.strip()
+
+            # Check if this is a stanza header
+            if stripped.startswith("[") and stripped.endswith("]"):
+                # If we were accumulating a previous stanza, finalize it first
+                if has_stanza_header and current_stanza_lines:
+                    stanzas.append("\n".join(current_stanza_lines))
+                    current_stanza_lines = []
+
+                # Start a new stanza
+                has_stanza_header = True
+                current_stanza_lines.append(line.rstrip())
+
+            # Skip comment lines
+            elif stripped.startswith("#"):
+                continue
+
+            # Blank line - finalize current stanza if we have one
+            elif stripped == "":
+                if has_stanza_header and current_stanza_lines:
+                    stanzas.append("\n".join(current_stanza_lines))
+                    current_stanza_lines = []
+                    has_stanza_header = False
+
+            # Non-empty, non-comment line - accumulate if we're in a stanza
+            else:
+                if has_stanza_header:
+                    current_stanza_lines.append(line.rstrip())
+
+    # Handle the last stanza if file doesn't end with a blank line
+    if has_stanza_header and current_stanza_lines:
+        stanzas.append("\n".join(current_stanza_lines))
+
+    return stanzas
+
+
+# NOTE: this will be removed once FBDs have proper support in contentctl-ng
+def append_savedsearches_fbd_conf(
+    app_factory: AppFactory, conf_directory: Path
+) -> None:
+    """Append content from static_configs/savedsearches_fbd.conf to savedsearches.conf.
+
+    This function reads the savedsearches_fbd.conf file from the content directory's
+    static_configs folder, parses it into stanzas, and appends them to the generated
+    savedsearches.conf file using a template format. This is done after the normal
+    conf generation because FBD searches are static conf content, not generated from
+    YML content objects.
+
+    Args:
+        app_factory: The app factory to use to locate the static_configs directory.
+        conf_directory: The directory containing the generated conf files.
+
+    Returns:
+        None - but as a result the savedsearches_fbd content is appended to savedsearches.conf
+    """
+    savedsearches_conf_path = conf_directory / "savedsearches.conf"
+
+    # Ensure the savedsearches.conf file exists before appending
+    if not savedsearches_conf_path.exists():
+        logger.warning(
+            f"savedsearches.conf does not exist at {savedsearches_conf_path}. "
+            "Cannot append savedsearches_fbd.conf content."
+        )
+        return
+
+    # Get the first content directory (there should be only one after porting)
+    content_directory = app_factory.app._content_directories[0]  # pyright: ignore[reportPrivateUsage]
+    fbd_conf_file = content_directory.path / "static_configs" / "savedsearches_fbd.conf"
+
+    if not fbd_conf_file.is_file():
+        logger.debug(f"No savedsearches_fbd.conf found at {fbd_conf_file}, skipping")
+        return
+
+    # Parse the FBD conf file into stanzas
+    fbd_stanzas = parse_savedsearches_fbd_conf(fbd_conf_file)
+
+    # Only write if there are actual stanzas
+    if not fbd_stanzas:
+        logger.debug("No FBD stanzas found in savedsearches_fbd.conf, skipping")
+        return
+
+    # Append to savedsearches.conf using template format
+    with open(savedsearches_conf_path, "a") as savedsearches_file:
+        for fbd_stanza in fbd_stanzas:
+            savedsearches_file.write("\n")
+            savedsearches_file.write(fbd_stanza)
+            savedsearches_file.write("\n")
+
+
+def build_directory_to_tar_gz(built_app_directory: Path) -> Path:
+    """Packages the app directory into an appropriate Splunk .tar.gz app archive.
+
+    Args:
+        built_app_directory: The directory containing the root of the built app.
+
+    Returns:
+        Path: The path to the built app archive in .tar.gz format
+    """
+    archive_path = built_app_directory.with_suffix(".tar.gz")
+    archive_path_latest = built_app_directory.with_name(
+        built_app_directory.name + "-latest"
+    ).with_suffix(".tar.gz")
+
+    logger.debug(f"Packaging app at {archive_path}")
+    logger.debug(
+        f"For legacy compatibility reasons, also packaging app at {archive_path_latest}"
+    )
+
+    # TODO: Should we keep both of these packages? Or, since the dist/
+    # Directory is always removed on build and the version is not longer
+    # part of the archive name, just emit the package name with no version
+    # or -latest on the end?
+    for arc_path in [archive_path, archive_path_latest]:
+        with tarfile.open(arc_path, "w:gz") as app_archive:
+            app_archive.add(
+                built_app_directory,
+                arcname=built_app_directory.name,
+            )
+        logger.info(f"Successfully packaged app at {arc_path}")
+
+    return archive_path
+
+
+def write_data_sources_csv(app_factory: AppFactory, scratch_directory: Path) -> None:
+    """Generate a CSV file about data_sources used by the app.
+
+    Args:
+        app_factory: The app factory to use to create the app.
+        scratch_directory: The directory to write the supporting files to.
+
+    Returns:
+        None - result is that the data_sources csv file is written to the scratch_directory
+    """
+    datasource_rows: list[dict[str, str]] = [
+        ds.datasource_csv_row for ds in app_factory.data_sources
+    ]
+
+    write_dynamic_csv(
+        app_factory, scratch_directory, "data_sources", datasource_rows, ["name"]
+    )
+
+
+def write_deprecaton_info_csv(app_factory: AppFactory, scratch_directory: Path) -> None:
+    """Generate a CSV file with info about deprecated content used by the app.
+
+    Args:
+        app_factory: The app factory to use to create the app.
+        scratch_directory: The directory to write the supporting files to.
+
+    Returns:
+        None - result is that the deprecation info csv file is written to the scratch_directory.
+    """
+    deprecation_info_rows: list[dict[str, str]] = []
+
+    # Macros and Schedules defined at runtime (e.g. _filter macros or schedules defined in detections)
+    # are included in this list of content,
+    # but they do not have deprecation_info
+    # so it's not an issue. This is one of the
+    # reasons we use hasattr rather than
+    # content.deprecation_info (field may not exist)
+    for content in app_factory.all_content_including_removed:
+        if (
+            isinstance(content, DeprecatableSecurityContent)
+            and content.deprecation_info_csv_row is not None
+        ):
+            deprecation_info_rows.append(content.deprecation_info_csv_row)
+    write_dynamic_csv(
+        app_factory,
+        scratch_directory,
+        "deprecation_info",
+        deprecation_info_rows,
+        ["Content Type", "Name"],
+    )
+
+
+def write_dynamic_csv(
+    app_factory: AppFactory,
+    scratch_directory: Path,
+    # TODO: Convert Consider conversion of Literal to StrEnum
+    dynamic_file_type: Literal["data_sources", "deprecation_info"],
+    rows: list[dict[str, str]],
+    sort_by_keys: list[str] = [],
+) -> None:
+    """Generate a CSV file dynamically for inclusion in the app."""
+    # These values are shared by all dynamic files
+    model_dict: dict[str, Any] = {
+        "author": app_factory.app._shared_app_settings.author,  # pyright: ignore[reportPrivateUsage]
+        "creation_date": app_factory.app._shared_app_settings._build_time.date(),  # pyright: ignore[reportPrivateUsage]
+        "modification_date": app_factory.app._shared_app_settings._build_time.date(),  # pyright: ignore[reportPrivateUsage]
+        "version": 1,
+    }
+
+    if dynamic_file_type == "data_sources":
+        fieldnames = [
+            "name",
+            "id",
+            "author",
+            "source",
+            "sourcetype",
+            "separator",
+            "supported_TA_name",
+            "supported_TA_version",
+            "supported_TA_url",
+            "description",
+        ]
+        model_dict.update(
+            {
+                "name": "data_sources",
+                "id": UUID(
+                    "b45c1403-6e09-47b0-824f-cf6e44f15ac8"
+                ),  # statically defined so it does not change
+                "description": "A lookup file that contains the data source objects for detections.",
+                "case_sensitive_match": False,
+            },
+        )
+
+    elif dynamic_file_type == "deprecation_info":
+        fieldnames = [
+            "Name",
+            "Content Type",
+            "Removed in Version",
+            "Reason",
+            "Replacement Content",
+            "Replacement Content Link",
+        ]
+        model_dict.update(
+            {
+                "name": "deprecation_info",
+                "id": UUID(
+                    "99262bf2-9606-4b52-b377-c96713527b35"
+                ),  # statically defined so it does not change
+                "description": "A lookup file that contains information about content that has been deprecated or removed from the app.",
+            }
+        )
+
+    else:
+        # Based on typing, we should never get here. But we keep this
+        # to be safe against future changes.
+        raise ValueError(f"Unknown dynamic file type: {dynamic_file_type}")
+
+    fake_yml_file_path = (scratch_directory / str(model_dict["name"])).with_suffix(
+        ".yml"
+    )
+    output_file_name = fake_yml_file_path.with_suffix(".csv")
+
+    # Generate the CSV File
+    # Due to how validation works for CSV Lookups Supporting CSV File,
+    # we MUST generate this first
+    # before we can create the CSVLookup object.
+    with open(
+        output_file_name,
+        "w",
+        newline="",
+    ) as f:
+        writer = DictWriter(
+            f,
+            fieldnames=fieldnames,
+            quoting=QUOTE_ALL,
+            extrasaction="raise",
+        )
+
+        # Write the header with titles for all the columns
+        writer.writeheader()
+        for row in sorted(rows, key=lambda x: tuple(x[k] for k in sort_by_keys)):
+            writer.writerow(row)
+
+    # Now that the CSV exists, we can create the CSVLookup object.
+
+    # TODO: The fact that _content_directories[0] is used is arbitrary.
+    # The constructor for a SecurityContent objects NEEDS a _content_directory
+    # in its context and, at this time:
+    # - all apps use the same settings
+    # - the Root app does not have its own ContentDirectory
+    # - there will always be at least one ContentDirectory (index 0)
+    # Should we ever introduce supporting multiple different build.yml and
+    # CustomAppSettings objects, we should revisit this choice.
+    # TODO: Consider whether these files should require a "content_directory" at
+    # all, or whether there should be a better approach for these types of runtime files.
+    context: dict[str, Any] = {
+        "_path": fake_yml_file_path,
+        # TODO: Very specifically, it is structurally and semantically "incorrect" to say that
+        # this content belongs to app_factory.app._content_directories[0] (which likely represents a repo,
+        # such as security_content_github or security_content_gitlab).
+        # It don't. It belongs at the "app" level rather than at the ContentDirectory level and
+        # is NOT backed a YML or CSV file that exists in a contentDirectory.
+        # However, we do not currently support the idea of "app-level content" rather than
+        # "ContentDirectory-level content". Take the requirement to build (and possibly track)
+        # "app-level content" in a meaningful way, and in addition to "ContentDirectory-level content",
+        # in future work.
+        "_content_dir": app_factory.app._content_directories[0],  # pyright: ignore[reportPrivateUsage]
+    }
+
+    dynamic_lookup = CSVLookup.model_validate(
+        model_dict,
+        context=context,
+    )
+    app_factory.csv_lookups.append(dynamic_lookup)
+
+
+def write_supporting_files(
+    app_factory: AppFactory, output_file_directory: Path
+) -> None:
+    """Some objects, such as Dashboards and CSVLookups, require additional files to be written to the app.
+
+    Args:
+        app_factory: The app factory to use to create the app.
+        output_file_directory: The directory to write the supporting files to.
+
+    Returns:
+        None - result is that supporting files are written to the output_file_directory
+        in the appropriate locations.
+    """
+    # Do not include removed content, since we shouldn't write supporting files for it.
+    for content_object in app_factory.all_content:
+        # We must check if the object is a SecurityContent object
+        # since this list also includes things like Macro
+        # and Schedule which MAY not SecurityContent objects.
+        # We could consider using a hasattr(content_object, "_supporting_file")
+        # instead, but I think isinstance is more explicit.
+        if (
+            isinstance(content_object, SecurityContent)  # pyright: ignore[reportUnnecessaryIsInstance] - added todo to app_factory.all_content to fix typing
+            and content_object._supporting_file  # pyright: ignore[reportPrivateUsage]
+        ):
+            content_object._supporting_file.write_to_file(output_file_directory)  # pyright: ignore[reportPrivateUsage]
+
+
+def build_app_to_directory(app_factory: AppFactory, build_directory: Path) -> Path:
+    """Creates the entire app structure, including static files, conf files, and tar.gz archive.
+
+    The build directory MUST not exist when we start building.
+
+    Args:
+        app_factory: The app factory to use to create the app.
+        build_directory: The directory to build the app to.
+
+    Returns:
+        Path: The path to the built app archive
+    """
+    if build_directory.exists():
+        # during development, we want to make sure we don't accidentally overwrite a directory
+        # that we didn't mean to. So let's make the deletion of the build directory something
+        # that requires explicit confirmation.
+        # This will be removed in the future.
+
+        logger.info(f"Deleting existing build directory {build_directory}")
+        shutil.rmtree(build_directory)
+
+    # Now make the directory which will actually contain the app
+    output_file_directory = build_directory / app_factory.app._shared_app_settings.id  # pyright: ignore[reportPrivateUsage]
+
+    # TODO - Writing these files to an arbitrary scratch_directory
+    # is likely not a good choice. We should consider whether these
+    # files must captured during a build process as artifacts and where
+    # to dump them in a more scalable and appropriate manner.
+    # Make a scratch directory for files that are dynamically generated
+    # during runtime.  At this time, this is only two files/pieces of content:
+
+    # - data_sources CSV and YML files
+    # - deprecation_info CSV and YML files
+    scratch_directory = build_directory / "scratch"
+    scratch_directory.mkdir(exist_ok=False, parents=True)
+
+    output_file_directory.mkdir(exist_ok=False, parents=True)
+
+    # write out all the components of the app
+    write_app_template(app_factory, output_file_directory)
+    write_data_sources_csv(app_factory, scratch_directory)
+    write_deprecaton_info_csv(app_factory, scratch_directory)
+    write_conf_files(app_factory, output_file_directory)
+    write_supporting_files(app_factory, output_file_directory)
+
+    logger.debug(
+        "MAKE SURE THAT THE MITRE CSV FILE HAS BEEN ADDED AS A YML AND "
+        "CSV INTO THE LOOKUPS DIRECTORY. IT CURRENTLY DOES NOT EXIST "
+        "AS STANDALONE CONTENT"
+    )
+
+    logger.info(f"Successfully built app at {output_file_directory}")
+    return output_file_directory
+
+
+def build_reporting_artifacts(app_factory: AppFactory, build_directory: Path) -> None:
+    """Builds reporting artifacts for the app.
+
+    This includes a number of SVG badges and the mitre attack json mapping file which
+    will power https://mitremap.splunkresearch.com/
+
+    Args:
+        app_factory (AppFactory): The app factory to use for building the app.
+        build_directory (Path): The directory to build the app in.
+
+    Returns:
+        None - but as a result the attack nav JSON file and SVG badges are created.
+    """
+    reporting_output_path = build_directory / "reporting"
+    try:
+        reporting_output_path.mkdir(parents=True, exist_ok=False)
+        logger.debug(f"Created reporting directory at {reporting_output_path}")
+    except FileExistsError:
+        raise ValueError(
+            f"Reporting output directory already exists: '{reporting_output_path}'"
+        )
+
+    # Read SVG Templates
+    # That these are just strings with format placeholders which we
+    # will update with the appropriate values
+
+    reporting_tools.build_svg_badges(app_factory.detections, reporting_output_path)
+    reporting_tools.build_mitre_attack_nav_json(
+        app_factory.detections, reporting_output_path
+    )
+    logger.info(f"Successfully wrote reporting information to {reporting_output_path}")
+
+
+class Builder(ContentSettings):
+    """Object that builds security content into a distributable format.
+
+    Args:
+        ContentSettings (ContentSettings): Settings for loading content.
+
+    """
+
+    model_config = SettingsConfigDict(
+        env_file=".env", extra="ignore", populate_by_name=True
+    )
+
+    def run(self) -> None:
+        """Builds the security content from content_dirs into the build output directory."""
+        build_path = build_app_to_directory(self.content, self.build_output_dir)
+        build_directory_to_tar_gz(build_path)
+        build_reporting_artifacts(self.content, self.build_output_dir)