s1-shift-left-cli 0.4.0__py3-none-any.whl

s1_cns_cli/s1graph/common/images/image_referencer.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from abc import abstractmethod
+from collections.abc import Iterable
+from pathlib import Path
+from typing import cast, Any, TYPE_CHECKING, Generic, TypeVar
+
+import aiohttp
+import docker
+
+from s1_cns_cli.s1graph.common.s1cns.vulnerability_scanning.image_scanner import image_scanner
+from s1_cns_cli.s1graph.common.s1cns.vulnerability_scanning.integrations.docker_image_scanning import \
+    docker_image_scanning_integration
+from s1_cns_cli.s1graph.common.output.common import ImageDetails
+from s1_cns_cli.s1graph.common.output.report import Report, CheckType
+from s1_cns_cli.s1graph.common.sca.commons import should_run_scan
+from s1_cns_cli.s1graph.common.sca.output import add_to_report_sca_data, get_license_statuses_async
+from s1_cns_cli.s1graph.common.typing import _LicenseStatus
+
+if TYPE_CHECKING:
+    from s1_cns_cli.s1graph.common.s1cns.platform_integration import BcPlatformIntegration
+    from s1_cns_cli.s1graph.runner_filter import RunnerFilter
+    from networkx import DiGraph
+
+_Definitions = TypeVar("_Definitions")
+
+INVALID_IMAGE_NAME_CHARS = ("[", "{", "(", "<", "$")
+
+
+def fix_related_resource_ids(report: Report | None, tmp_dir: str) -> None:
+    """Remove tmp dir prefix from 'relatedResourceId'"""
+
+    if report and report.image_cached_results:
+        for cached_result in report.image_cached_results:
+            related_resource_id = cached_result.get("relatedResourceId")
+            if related_resource_id and isinstance(related_resource_id, str):
+                cached_result["relatedResourceId"] = related_resource_id.replace(tmp_dir, "", 1)
+
+
+class Image:
+    def __init__(self, file_path: str, name: str, start_line: int, end_line: int,
+                 related_resource_id: str | None = None) -> None:
+        """
+
+        :param file_path: example: 's1cns/integration_tests/example_workflow_file/.github/workflows/vulnerable_container.yaml'
+        :param name: example: 'node:14.16'
+        :param image_id: example: 'sha256:6a353e22ce'
+        :param start_line: example: 8
+        :param end_line: example: 16
+        """
+        self.end_line = end_line
+        self.start_line = start_line
+        self.name = name
+        self.file_path = file_path
+        self.related_resource_id = related_resource_id
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, self.__class__):
+            return self.__dict__ == other.__dict__
+
+        return False
+
+    def __ne__(self, other: Any) -> bool:
+        return not self.__eq__(other)
+
+    def __hash__(self) -> int:
+        return hash((self.file_path, self.name, self.start_line, self.end_line))
+
+
+class ImageReferencer:
+    @abstractmethod
+    def is_workflow_file(self, file_path: str) -> bool:
+        """
+
+        :param file_path: path of file to validate if it is a file that contains might images (example: CI workflow file)
+        :return: True if contains images
+
+        """
+        return False
+
+    @abstractmethod
+    def get_images(self, file_path: str) -> Iterable[Image]:
+        """
+        Get container images mentioned in a file
+        :param file_path: File to be inspected
+        :return: List of container images objects mentioned in the file.
+        """
+        return []
+
+    @staticmethod
+    def inspect(image_name: str) -> str:
+        """
+
+        :param image_name: name of the image to be inspected locally using a "docker inspect X". If image does not exist try to pull it locally.
+        :return: short image id sha that is inspected. In case inspect has failed None will be returned.
+        """
+        try:
+            logging.info("Inspecting docker image {}".format(image_name))
+            client = docker.from_env()
+            try:
+                image = client.images.get(image_name)
+            except Exception:
+                image = client.images.pull(image_name)
+                return cast(str, image.short_id)
+            return cast(str, image.short_id)
+        except Exception:
+            logging.debug(f"failed to pull docker image={image_name}", exc_info=True)
+            return ""
+
+
+def is_valid_public_image_name(image_name: str) -> bool:
+    if image_name.startswith('localhost'):
+        return False
+    if any(char in image_name for char in INVALID_IMAGE_NAME_CHARS):
+        return False
+    if image_name.count(":") > 1:
+        # if there is more than one colon, then it is typically a private registry with port reference
+        return False
+    return True
+
+
+class ImageReferencerMixin(Generic[_Definitions]):
+    """Mixin class to simplify image reference search"""
+
+    def check_container_image_references(
+        self,
+        root_path: str | Path | None,
+        runner_filter: RunnerFilter,
+        graph_connector: DiGraph | None = None,
+        definitions: _Definitions | None = None,
+        definitions_raw: dict[str, list[tuple[int, str]]] | None = None,
+    ) -> Report | None:
+        """Tries to find image references in graph based IaC templates"""
+        from s1_cns_cli.s1graph.common.s1cns.platform_integration import bc_integration
+
+        # skip complete run, if flag '--check' was used without a CVE check ID
+        if not should_run_scan(runner_filter.checks):
+            return None
+
+        images = self.extract_images(graph_connector=graph_connector, definitions=definitions,
+                                     definitions_raw=definitions_raw)
+        if not images:
+            return None
+
+        logging.info(f"Found {len(images)} image references {[image.name for image in images]}")
+
+        report = Report(CheckType.SCA_IMAGE)
+        root_path = Path(root_path) if root_path else None
+        check_class = f"{image_scanner.__module__}.{image_scanner.__class__.__qualname__}"
+        report_type = CheckType.SCA_IMAGE
+        image_names_to_query = list(set(filter(lambda i: is_valid_public_image_name(i), map(lambda i: i.name, images))))
+        results = asyncio.run(self._fetch_image_results_async(image_names_to_query))
+
+        license_statuses_by_image = asyncio.run(self._fetch_licenses_per_image(image_names_to_query, results))
+
+        for image in images:
+            try:
+                results_index = image_names_to_query.index(image.name)
+                cached_results = results[results_index]
+            except ValueError:
+                cached_results = {}
+
+            file_line_range = [image.start_line, image.end_line]
+            self._add_image_records(
+                report=report,
+                root_path=root_path,
+                check_class=check_class,
+                dockerfile_path=image.file_path,
+                image=image,
+                runner_filter=runner_filter,
+                report_type=report_type,
+                bc_integration=bc_integration,
+                cached_results=cached_results,
+                license_statuses=license_statuses_by_image.get(image.name) or [],
+                file_line_range=file_line_range if None not in file_line_range else None
+            )
+
+        return report
+
+    @staticmethod
+    async def _fetch_image_results_async(image_names_to_query: list[str]) -> list[dict[str, Any]]:
+        """
+        This is an async implementation of `_fetch_image_results`. The only change is we're getting a session
+        as an input, and the asyncio behavior is managed in the calling method.
+        """
+        async with aiohttp.ClientSession() as session:
+            results: list[dict[str, Any]] = await asyncio.gather(*[
+                image_scanner.get_scan_results_from_cache_async(session, f"image:{i}")
+                for i in image_names_to_query
+            ])
+        return results
+
+    def _add_image_records(
+        self,
+        report: Report,
+        root_path: Path | None,
+        check_class: str,
+        dockerfile_path: str,
+        image: Image,
+        runner_filter: RunnerFilter,
+        report_type: str,
+        bc_integration: BcPlatformIntegration,
+        cached_results: dict[str, Any],
+        license_statuses: list[_LicenseStatus],
+        file_line_range: list[int] | None = None
+    ) -> None:
+        """Adds an image record to the given report, if possible"""
+        if cached_results:
+            logging.info(f"Found cached scan results of image {image.name}")
+            image_scanning_report: dict[str, Any] = docker_image_scanning_integration.create_report(
+                twistcli_scan_result=cached_results,
+                bc_platform_integration=bc_integration,
+                file_path=dockerfile_path,
+                file_content=f'image: {image.name}',
+                docker_image_name=image.name,
+                related_resource_id=image.related_resource_id,
+                root_folder=root_path,
+                error_lines=file_line_range
+            )
+            report.image_cached_results.append(image_scanning_report)
+
+            result = cached_results.get("results", [{}])[0]
+            image_id = self._extract_image_short_id(result)
+            image_details = self._get_image_details_from_twistcli_result(scan_result=result, image_id=image_id)
+            dockerfile_rel_path = dockerfile_path
+            if root_path:
+                try:
+                    dockerfile_rel_path = str(Path(dockerfile_path).relative_to(root_path))
+                except ValueError:
+                    # Path.is_relative_to() was implemented in Python 3.9
+                    pass
+            rootless_file_path = dockerfile_rel_path.replace(Path(dockerfile_rel_path).anchor, "", 1)
+            rootless_file_path_to_report = f"{rootless_file_path} ({image.name} lines:{image.start_line}-" \
+                                           f"{image.end_line} ({image_id}))"
+
+            self._add_vulnerability_records(
+                report=report,
+                result=result,
+                check_class=check_class,
+                dockerfile_path=dockerfile_path,
+                rootless_file_path=rootless_file_path_to_report,
+                image_details=image_details,
+                runner_filter=runner_filter,
+                report_type=report_type,
+                license_statuses=license_statuses,
+                file_line_range=file_line_range
+            )
+        else:
+            logging.info(f"No cache hit for image {image.name}")
+
+    @staticmethod
+    def _extract_image_short_id(scan_result: dict[str, Any]) -> str:
+        """Extracts a shortened version of the image ID from the scan result"""
+
+        if "id" not in scan_result:
+            return "sha256:unknown"
+
+        image_id: str = scan_result["id"]
+
+        if image_id.startswith("sha256:"):
+            return image_id[:17]
+        return image_id[:10]
+
+    @staticmethod
+    def _get_image_details_from_twistcli_result(scan_result: dict[str, Any], image_id: str) -> ImageDetails:
+        """Extracts the image detaisl from a twistcli scan result"""
+
+        image_packages = scan_result.get("packages", [])
+        image_package_types = {f'{package["name"]}@{package["version"]}': package["type"] for package in image_packages}
+        return ImageDetails(
+            distro=scan_result.get("distro", ""),
+            distro_release=scan_result.get("distroRelease", ""),
+            package_types=image_package_types,
+            image_id=image_id,
+        )
+
+    @staticmethod
+    def _add_vulnerability_records(
+        report: Report,
+        result: dict[str, Any],
+        check_class: str,
+        dockerfile_path: str,
+        rootless_file_path: str,
+        image_details: ImageDetails | None,
+        license_statuses: list[_LicenseStatus],
+        runner_filter: RunnerFilter,
+        report_type: str,
+        file_line_range: list[int] | None = None
+    ) -> None:
+        vulnerabilities = result.get("vulnerabilities", [])
+        packages = result.get("packages", [])
+        add_to_report_sca_data(
+            report=report,
+            check_class=check_class,
+            scanned_file_path=dockerfile_path,
+            rootless_file_path=rootless_file_path,
+            runner_filter=runner_filter,
+            vulnerabilities=vulnerabilities,
+            packages=packages,
+            license_statuses=license_statuses,
+            sca_details=image_details,
+            report_type=report_type,
+            file_line_range=file_line_range
+        )
+
+    @abstractmethod
+    def extract_images(
+        self,
+        graph_connector: DiGraph | None = None,
+        definitions: _Definitions | None = None,
+        definitions_raw: dict[str, list[tuple[int, str]]] | None = None
+    ) -> list[Image]:
+        """Tries to find image references in the graph or supported resource"""
+
+        pass
+
+    @staticmethod
+    async def _fetch_licenses_per_image(image_names: list[str], image_results: list[dict[str, Any]]) \
+            -> dict[str, list[_LicenseStatus]]:
+        merged_result: dict[str, list[_LicenseStatus]] = {}
+        async with aiohttp.ClientSession() as session:
+            license_results = await asyncio.gather(*[
+                get_license_statuses_async(session, result['results'][0].get('packages') or [], image_names[i])
+                for i, result in enumerate(image_results)
+                if "results" in result and result["results"]
+            ])
+        merged_result.update({r['image_name']: r['licenses'] for r in license_results})
+        return merged_result

s1_cns_cli/s1graph/common/images/workflow/image_referencer_manager.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from s1_cns_cli.s1graph.common.images.workflow.image_referencer_provider import WorkflowImageReferencerProvider
+
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from s1_cns_cli.s1graph.common.images.image_referencer import Image
+
+
+class WorkflowImageReferencerManager:
+    __slots__ = ("workflow_config", "file_path", "provider")
+
+    def __init__(self, workflow_config: dict[str, Any], file_path: str, provider: WorkflowImageReferencerProvider):
+        self.workflow_config = workflow_config
+        self.file_path = file_path
+        self.provider = provider
+
+    def extract_images_from_workflow(self) -> list[Image]:
+        images: list[Image] = self.provider.extract_images_from_workflow()
+        return images

s1_cns_cli/s1graph/common/images/workflow/image_referencer_provider.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Any
+
+from s1_cns_cli.s1graph.common.images.image_referencer import Image
+from s1_cns_cli.s1graph.common.util.consts import START_LINE, END_LINE
+
+
+class WorkflowImageReferencerProvider:
+    def __init__(self, workflow_config: dict[str, Any], file_path: str):
+        self.workflow_config = workflow_config
+        self.file_path = file_path
+
+    @staticmethod
+    def _get_start_end_lines(entity: dict[str, Any]) -> tuple[int, int]:
+        return entity.get(START_LINE, 0), entity.get(END_LINE, 0)
+
+    @abstractmethod
+    def extract_images_from_workflow(self) -> list[Image]:
+        pass

s1_cns_cli/s1graph/common/models/consts.py

@@ -0,0 +1,35 @@
+import re
+
+SUPPORTED_FILE_EXTENSIONS = [".tf", ".yml", ".yaml", ".json", ".template", ".bicep", ".hcl"]
+SUPPORTED_PACKAGE_FILES = {
+    "bower.json",
+    "build.gradle",
+    "build.gradle.kts",
+    "go.sum",
+    "gradle.properties",
+    "METADATA",
+    "npm-shrinkwrap.json",
+    "package.json",
+    "package-lock.json",
+    "pom.xml",
+    "requirements.txt"
+}
+SUPPORTED_FILES = SUPPORTED_PACKAGE_FILES.union({"Dockerfile"})
+
+DEPENDENCY_TREE_SUPPORTED_FILES = {"yarn.lock", "Gemfile", "Gemfile.lock", "go.mod", "paket.dependencies", "paket.lock", "packages.config"}
+
+SCANNABLE_PACKAGE_FILES_EXTENSIONS = {".csproj"}
+
+SCANNABLE_PACKAGE_FILES = SUPPORTED_PACKAGE_FILES | DEPENDENCY_TREE_SUPPORTED_FILES
+
+ANY_VALUE = "CKV_ANY"
+DOCKER_IMAGE_REGEX = re.compile(r'(?:[^\s\/]+\/)?([^\s:]+):?([^\s]*)')
+access_key_pattern = re.compile("(?<![A-Z0-9])[A-Z0-9]{20}(?![A-Z0-9])")  # nosec
+secret_key_pattern = re.compile("(?<![A-Za-z0-9/+=])[A-Za-z0-9/+=]{40}(?![A-Za-z0-9/+=])")  # nosec
+linode_token_pattern = re.compile("(?<![A-Za-z0-9/+=])[A-Za-z0-9/+=]{64}(?![A-Za-z0-9/+=])")  # nosec
+s1cns_token_pattern = re.compile(r"^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z")  # nosec
+panos_api_key_pattern = re.compile(r"^LUFRPT1[a-zA-Z0-9]+==\Z")  # nosec
+SLS_DEFAULT_VAR_PATTERN = re.compile(r"\${([^{}]+?)}")
+YAML_COMMENT_MARK = '#'
+TFC_HOST_NAME = "app.terraform.io"
+ckv_check_id_pattern = re.compile(r"^CKV2?_[A-Za-z]*_[0-9]*$")

s1_cns_cli/s1graph/common/models/enums.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from enum import Enum, IntEnum
+
+
+class CheckResult(str, Enum):
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    # Unknown should be used when a check does not wish to return a result, generally due to the inability
+    # to resolve a value or similar types of errors.
+    UNKNOWN = "UNKNOWN"
+    # Skipped is used by the framework when a test is suppressed and should not be used directly by checks.
+    SKIPPED = "SKIPPED"
+
+
+class CheckCategories(Enum):
+    LOGGING = 1
+    ENCRYPTION = 2
+    GENERAL_SECURITY = 3
+    NETWORKING = 4
+    IAM = 5
+    BACKUP_AND_RECOVERY = 6
+    CONVENTION = 7
+    SECRETS = 8
+    KUBERNETES = 9
+    APPLICATION_SECURITY = 10
+    SUPPLY_CHAIN = 11
+    API_SECURITY = 12
+
+
+class OutputFormat(Enum):
+    CONSOLE = 1
+    JSON = 2
+    JUNIT_XML = 3
+
+
+class ContextCategories(Enum):
+    PROVIDER = 1
+    TERRAFORM = 2
+    LOCALS = 3
+    MODULE = 4
+    DATA = 5
+    RESOURCE = 6
+    VARIABLE = 7
+    OUTPUT = 8
+
+
+class ScanDataFormat(Enum):
+    TWISTCLI = 1
+    PLATFORM = 2
+    DEPENDENCY_TREE = 3
+
+
+class ErrorStatus(IntEnum):
+    SUCCESS = 0
+    ERROR = 2
+
+
+@dataclass
+class CheckFailLevel:
+    WARNING = 'WARNING'
+    ERROR = 'ERROR'

s1_cns_cli/s1graph/common/multi_signature.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import inspect
+from abc import ABCMeta
+from functools import update_wrapper
+from types import CodeType
+from typing import Callable, Any, TypeVar, cast
+from typing_extensions import Protocol
+
+_MultiT = TypeVar("_MultiT")
+
+
+class _MultiSignataureMethod(Protocol):
+    __code__: CodeType
+    __multi_signature_wrappers__: dict[tuple[tuple[str, ...], Any, Any], Callable[..., Any]]
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        ...
+
+    def add_signature(self, *, args: list[str], varargs: Any = None, varkw: Any = None) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+        ...
+
+
+class MultiSignatureMeta(ABCMeta):  # noqa: B024  # needs to be ABCMeta, because of the super().__new__ call
+    __multi_signature_methods__: dict[str, _MultiSignataureMethod]  # noqa: CCE003
+
+    def __new__(cls, name: str, bases: tuple[Any], namespace: dict[str, Any], **kwargs: Any) -> MultiSignatureMeta:
+        mcs = super().__new__(cls, name, bases, namespace, **kwargs)
+        multi_signatures: dict[str, _MultiSignataureMethod] = {
+            name: value  # type:ignore[misc]
+            for name, value in namespace.items()
+            if hasattr(value, "__multi_signature_wrappers__") and inspect.isfunction(value)
+            # isfunction, because function is not bound yet
+        }
+        # search in base classes for decorated functions
+        for base in bases:
+            for name, value in getattr(base, "__multi_signature_methods__", {}).items():
+                if inspect.isfunction(value) and hasattr(value, "__multi_signature_wrappers__"):
+                    multi_signature_wrappers = getattr(value, "__multi_signature_wrappers__", {})
+                    if multi_signature_wrappers:
+                        current_function = multi_signatures.get(name)
+                        if current_function:
+                            current_function.__multi_signature_wrappers__.update(multi_signature_wrappers)
+                        else:
+                            multi_signatures[name] = cast(_MultiSignataureMethod, value)
+
+        mcs.__multi_signature_methods__ = multi_signatures
+        for name, value in multi_signatures.items():
+            wrapped = getattr(mcs, name)
+            arguments = inspect.getargs(wrapped.__code__)
+            if arguments == inspect.getargs(value.__code__):
+                # Do not replace if the signature is the same
+                continue
+            # convert args into a tuple
+            args, varargs, varkw = arguments
+            multi_signature_key = tuple(args), varargs, varkw
+            get_wrapper = value.__multi_signature_wrappers__.get(tuple(multi_signature_key), None)
+            if get_wrapper:
+                wrapper = get_wrapper(mcs, wrapped)
+                update_wrapper(wrapper, wrapped)
+                setattr(mcs, name, wrapper)
+            else:
+                # unknown implementation
+                raise NotImplementedError(f"The signature {multi_signature_key} for {name} is not supported.")
+
+        return mcs
+
+
+class multi_signature:
+    """
+    Decorator to allow other signatures in sub classes.
+
+    You can use this decorator on class methods. The implementation requires that the metaclass of that class is
+    :class:`MultiSignatureMeta`
+    This class extends :class:`ABCMeta` so it supports abstract base classes.
+    """
+
+    def __init__(self) -> None:
+        self.__wrappers__: dict[tuple[tuple[str, ...], Any, Any], Callable[..., _MultiT]] = {}
+
+    def __call__(self, fn: Callable[..., _MultiT]) -> _MultiSignataureMethod:
+        fn.add_signature = self.add_signature  # type:ignore[attr-defined]
+        fn.__multi_signature_wrappers__ = self.__wrappers__  # type:ignore[attr-defined]
+        return cast(_MultiSignataureMethod, fn)
+
+    def add_signature(
+        self, *, args: list[str], varargs: Any = None, varkw: Any = None
+    ) -> Callable[[Callable[..., _MultiT]], Callable[..., _MultiT]]:
+        """
+        Registers a new wrapper for the decorated function.
+
+        The decorated function must have two parameters (`cls` and `wrapped`) (The names are not enforced.).
+        The first one is the class so this has to be a class function. The second one is the function to wrap.
+        It has to return a function that has the same signature as the one decorated with `@multi_signature`.
+        The returned function has to call `wrapped`, which is a function with the parameters listed as `args`
+        and must return the result of `wrapped`. If the wrapped function has varargs, the name of this variable
+        is specified with `varargs`. For keyword args it is `varkw`. See :func:`inspect.getargs` to get the
+        correct order.
+
+        The implementation must not handle doc preservation. This is already done by the implementation.
+
+        :param args: the parameters that the wrapped version of the function accepts.
+        :param varargs: if the wrapped version uses varargs, this is the name of this
+            parameter. If not used, this is `None`.
+        :param varkw: if the wrapped version uses keyword arguments, this is the name
+            of this parameter. If not used, this is `None`.
+
+        :Example:
+
+        >>> class Example(metaclass=MultiSignatureMeta:
+        >>>
+        >>>    @multi_signature
+        >>>    def some_function(self, a, b, c):
+        >>>        pass
+        >>>
+        >>>    @classmethod
+        >>>    @some_function.add_signature("self", "a")
+        >>>    def _some_function_self_a(cls, wrapped):
+        >>>        def wrapper(self, a, b, c):
+        >>>            return wrapped(self, a)
+        >>>
+        >>>         return wrapper
+        """
+
+        def wrapper(fn: Callable[..., _MultiT]) -> Callable[..., _MultiT]:
+            self.__wrappers__[(tuple(args), varargs, varkw)] = fn  # type:ignore[assignment]  # mypy bug
+            return fn
+
+        return wrapper