s1-shift-left-cli 0.4.0__py3-none-any.whl

s1_cns_cli/s1graph/cloudformation/image_referencer/provider/aws.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from s1_cns_cli.s1graph.cloudformation.image_referencer.base_provider import BaseCloudFormationProvider
+from s1_cns_cli.s1graph.common.util.data_structures_utils import find_in_dict
+from s1_cns_cli.s1graph.common.util.type_forcers import extract_json
+
+if TYPE_CHECKING:
+    from networkx import DiGraph
+
+
+class AwsCloudFormationProvider(BaseCloudFormationProvider):
+    def __init__(self, graph_connector: DiGraph) -> None:
+        super().__init__(
+            graph_connector=graph_connector,
+            supported_resource_types=SUPPORTED_AWS_IMAGE_RESOURCE_TYPES,
+        )
+
+
+def extract_images_from_aws_apprunner_service(resource: dict[str, Any]) -> list[str]:
+    image_names: list[str] = []
+
+    image_repo = find_in_dict(input_dict=resource, key_path="SourceConfiguration/ImageRepository")
+    if isinstance(image_repo, dict):
+        repo_type = image_repo.get("ImageRepositoryType")
+        name = image_repo.get("ImageIdentifier")
+        if name and isinstance(name, str) and repo_type == "ECR_PUBLIC":
+            image_names.append(name)
+
+    return image_names
+
+
+def extract_images_from_aws_batch_job_definition(resource: dict[str, Any]) -> list[str]:
+    image_names: list[str] = []
+
+    properties = extract_json(resource.get("ContainerProperties"))
+    if isinstance(properties, dict):
+        name = properties.get("Image")
+        if name and isinstance(name, str):
+            image_names.append(name)
+
+    node_range = find_in_dict(input_dict=resource, key_path="NodeProperties/NodeRangeProperties")
+    if isinstance(node_range, list):
+        for node in node_range:
+            name = find_in_dict(input_dict=node, key_path="Container/Image")
+            if name and isinstance(name, str):
+                image_names.append(name)
+
+    return image_names
+
+
+def extract_images_from_aws_codebuild_project(resource: dict[str, Any]) -> list[str]:
+    image_names: list[str] = []
+
+    name = find_in_dict(input_dict=resource, key_path="Environment/Image")
+    if name and isinstance(name, str):
+        # AWS provided images have an internal identifier
+        if not name.startswith("aws/codebuild/"):
+            image_names.append(name)
+
+    return image_names
+
+
+def extract_images_from_aws_ecs_task_definition(resource: dict[str, Any]) -> list[str]:
+    image_names: list[str] = []
+
+    definitions = extract_json(resource.get("ContainerDefinitions"))
+    if isinstance(definitions, list):
+        for definition in definitions:
+            name = definition.get("Image")
+            if name and isinstance(name, str):
+                image_names.append(name)
+
+    return image_names
+
+
+def extract_images_from_aws_lightsail_container_service_deployment_version(resource: dict[str, Any]) -> list[str]:
+    image_names: list[str] = []
+
+    containers = find_in_dict(input_dict=resource, key_path="ContainerServiceDeployment/Containers")
+    if isinstance(containers, list):
+        for container in containers:
+            name = container.get("Image")
+            if name and isinstance(name, str):
+                image_names.append(name)
+
+    return image_names
+
+
+# needs to be at the bottom to add the defined functions
+SUPPORTED_AWS_IMAGE_RESOURCE_TYPES = {
+    "AWS::AppRunner::Service": extract_images_from_aws_apprunner_service,
+    "AWS::Batch::JobDefinition": extract_images_from_aws_batch_job_definition,
+    "AWS::CodeBuild::Project": extract_images_from_aws_codebuild_project,
+    "AWS::ECS::TaskDefinition": extract_images_from_aws_ecs_task_definition,
+    "AWS::Lightsail::Container": extract_images_from_aws_lightsail_container_service_deployment_version,
+}

s1_cns_cli/s1graph/cloudformation/parser/__init__.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import logging
+from typing import Dict, Optional, Any
+
+from s1_cns_cli.s1graph.cloudformation.parser import cfn_yaml
+from s1_cns_cli.s1graph.common.parsers.json import parse as json_parse
+from s1_cns_cli.s1graph.cloudformation.parser.cfn_keywords import TemplateSections
+from yaml.scanner import ScannerError
+from yaml import YAMLError
+
+LOGGER = logging.getLogger(__name__)
+
+
+def parse(
+    filename: str, out_parsing_errors: Optional[Dict[str, str]] = None
+) -> tuple[dict[str, Any] | list[dict[str, Any]], list[tuple[int, str]]] | tuple[None, None]:
+    """
+    Decode filename into an object
+    """
+    template: "dict[str, Any] | list[dict[str, Any]] | None" = None
+    template_lines = None
+    error = None
+
+    if out_parsing_errors is None:
+        out_parsing_errors = {}
+
+    try:
+        (template, template_lines) = cfn_yaml.load(filename, cfn_yaml.ContentType.CFN)
+    except IOError as err:
+        if err.errno == 2:
+            error = f"Template file not found: {filename} - {err}"
+            LOGGER.error(error)
+        elif err.errno == 21:
+            error = f"Template references a directory, not a file: {filename} - {err}"
+            LOGGER.error(error)
+        elif err.errno == 13:
+            error = f"Permission denied when accessing template file: {filename} - {err}"
+            LOGGER.error(error)
+    except UnicodeDecodeError as err:
+        error = f"Cannot read file contents: {filename} - {err}"
+        LOGGER.error(error)
+    except cfn_yaml.CfnParseError as err:
+        if "Null value at" in err.message:
+            LOGGER.info(f"Null values do not exist in CFN templates: {filename} - {err}")
+            return None, None
+
+        error = f"Parsing error in file: {filename} - {err}"
+        LOGGER.info(error)
+    except ValueError as err:
+        error = f"Parsing error in file: {filename} - {err}"
+        LOGGER.info(error)
+    except ScannerError as err:
+        if err.problem in ["found character '\\t' that cannot start any token", "found unknown escape character"]:
+            try:
+                result = json_parse(filename, allow_nulls=False)
+                if result:
+                    template, template_lines = result
+            except Exception as json_err:  # pylint: disable=W0703
+                error = f"Template {filename} is malformed: {err.problem}. Tried to parse {filename} as JSON but got error: {json_err}"
+                LOGGER.info(error)
+    except YAMLError as err:
+        if hasattr(err, 'problem') and err.problem in ["expected ',' or '}', but got '<scalar>'"]:
+            try:
+                result = json_parse(filename, allow_nulls=False)
+                if result:
+                    template, template_lines = result
+            except Exception as json_err:  # pylint: disable=W0703
+                error = f"Template {filename} is malformed: {err.problem}. Tried to parse {filename} as JSON but got error: {json_err}"
+                LOGGER.info(error)
+        else:
+            error = f"Parsing error in file: {filename} - {err}"
+            LOGGER.info(error)
+
+    if error:
+        out_parsing_errors[filename] = error
+
+    if isinstance(template, dict):
+        resources = template.get(TemplateSections.RESOURCES.value, None)
+        if resources and isinstance(resources, dict):
+            if "__startline__" in resources:
+                del resources["__startline__"]
+            if "__endline__" in resources:
+                del resources["__endline__"]
+
+    if template is None or template_lines is None:
+        return None, None
+
+    return template, template_lines

s1_cns_cli/s1graph/cloudformation/parser/cfn_keywords.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+@dataclass
+class IntrinsicFunctions:
+    BASE64 = "Fn::Base64"
+    CIDR = "Fn::Cidr"
+    FIND_IN_MAP = "Fn::FindInMap"
+    GET_ATT = "Fn::GetAtt"
+    GET_AZS = "Fn::GetAZs"
+    IMPORT_VALUE = "Fn::ImportValue"
+    JOIN = "Fn::Join"
+    SELECT = "Fn::Select"
+    SPLIT = "Fn::Split"
+    SUB = "Fn::Sub"
+    TRANSFORM = "Fn::Transform"
+    REF = "Ref"
+    CONDITION = "Condition"
+
+
+@dataclass
+class ConditionFunctions:
+    AND = "Fn::And"
+    EQUALS = "Fn::Equals"
+    IF = "Fn::If"
+    NOT = "Fn::Not"
+    OR = "Fn::Or"
+
+
+@dataclass
+class ResourceAttributes:
+    CREATION_POLICY = "CreationPolicy"
+    DELETION_POLICY = "DeletionPolicy"
+    DEPENDS_ON = "DependsOn"
+    METADATA = "Metadata"
+    UPDATE_POLICY = "UpdatePolicy"
+    UPDATE_REPLACE_POLICY = "UpdateReplacePolicy"
+
+
+class TemplateSections(str, Enum):
+    RESOURCES = "Resources"
+    METADATA = "Metadata"
+    PARAMETERS = "Parameters"
+    RULES = "Rules"
+    MAPPINGS = "Mappings"
+    CONDITIONS = "Conditions"
+    TRANSFORM = "Transform"
+    OUTPUTS = "Outputs"
+    GLOBALS = "Globals"

s1_cns_cli/s1graph/cloudformation/parser/cfn_yaml.py

@@ -0,0 +1,286 @@
+"""
+Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: MIT-0
+"""
+from __future__ import annotations
+
+import json
+import logging
+import platform
+from collections.abc import Hashable
+from enum import Enum
+from pathlib import Path
+from typing import Any, TYPE_CHECKING, NoReturn, Callable
+
+from yaml import MappingNode
+from yaml import ScalarNode
+from yaml import SequenceNode
+from yaml.composer import Composer
+from yaml.constructor import ConstructorError
+from yaml.constructor import SafeConstructor
+from yaml.reader import Reader
+from yaml.resolver import Resolver
+from yaml.scanner import Scanner
+from charset_normalizer import from_path
+
+from s1_cns_cli.s1graph.common.parsers.json.decoder import SimpleDecoder
+from s1_cns_cli.s1graph.common.parsers.node import StrNode, DictNode, ListNode
+from s1_cns_cli.s1graph.common.util.consts import MAX_IAC_FILE_SIZE
+
+try:
+    from yaml.cyaml import CParser as Parser  # type:ignore[attr-defined]
+
+    cyaml = True
+except ImportError:
+    from yaml.parser import Parser  # type:ignore[assignment]
+
+    cyaml = False
+
+if TYPE_CHECKING:
+    from yaml import Node
+
+UNCONVERTED_SUFFIXES = ['Ref', 'Condition']
+FN_PREFIX = 'Fn::'
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ContentType(str, Enum):
+    CFN = "CFN"
+    SLS = "SLS"
+    TFPLAN = "TFPLAN"
+
+
+class CfnParseError(ConstructorError):
+    """
+    Error thrown when the template contains Cfn Error
+    """
+
+    def __init__(self, filename: str, message: str, line_number: int, column_number: int) -> None:
+        # Call the base class constructor with the parameters it needs
+        super(CfnParseError, self).__init__(message)
+
+        # Now for your custom codescanner...
+        self.filename = filename
+        self.line_number = line_number
+        self.column_number = column_number
+        self.message = message
+
+
+class NodeConstructor(SafeConstructor):
+    """
+    Node Constructors for loading different types in Yaml
+    """
+
+    def __init__(self, filename: str, content_type: ContentType | None = None) -> None:
+        # Call the base class constructor
+        super().__init__()
+        self.add_constructor(  # type:ignore[type-var]
+            u'tag:yaml.org,2002:map',
+            NodeConstructor.construct_yaml_map,
+        )
+
+        self.add_constructor(  # type:ignore[type-var]
+            u'tag:yaml.org,2002:str',
+            NodeConstructor.construct_yaml_str,
+        )
+
+        self.add_constructor(  # type:ignore[type-var]
+            u'tag:yaml.org,2002:seq',
+            NodeConstructor.construct_yaml_seq,
+        )
+        if content_type != ContentType.TFPLAN:
+            NodeConstructor.add_constructor(  # type:ignore[type-var]
+                u'tag:yaml.org,2002:null',
+                NodeConstructor.construct_yaml_null_error,
+            )
+        self.filename = filename
+
+    # To support lazy loading, the original constructors first yield
+    # an empty object, then fill them in when iterated. Due to
+    # laziness we omit this behaviour (and will only do "deep
+    # construction") by first exhausting iterators, then yielding
+    # copies.
+    def construct_yaml_map(self, node: MappingNode) -> DictNode:
+        # Check for duplicate keys on the current level, this is not desirable
+        # because a dict does not support this. It overwrites it with the last
+        # occurance, which can give unexpected results
+        mapping = {}
+        self.flatten_mapping(node)
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, False)  # type:ignore[no-untyped-call]
+            value = self.construct_object(value_node, False)  # type:ignore[no-untyped-call]
+            try:
+                if isinstance(key, dict):
+                    key = frozenset(key.keys()), frozenset(key.values())
+                if isinstance(key, list):
+                    key = frozenset(key)
+            except TypeError:
+                raise CfnParseError(
+                    filename=self.filename,
+                    message=f'Unable to construct key {key} (line {key_node.start_mark.line + 1})',
+                    line_number=key_node.start_mark.line,
+                    column_number=key_node.start_mark.column,
+                ) from None
+            if key in mapping:
+                raise CfnParseError(
+                    filename=self.filename,
+                    message=f'Duplicate resource found "{key}" (line {key_node.start_mark.line + 1})',
+                    line_number=key_node.start_mark.line,
+                    column_number=key_node.start_mark.column,
+                )
+            mapping[key] = value
+
+        obj, = SafeConstructor.construct_yaml_map(self, node)  # type:ignore[no-untyped-call]
+        return DictNode(obj, node.start_mark, node.end_mark)
+
+    def construct_yaml_str(self, node: ScalarNode) -> StrNode:
+        obj = SafeConstructor.construct_yaml_str(self, node)  # type:ignore[no-untyped-call]
+        assert isinstance(obj, str)  # nosec
+        return StrNode(obj, node.start_mark, node.end_mark)
+
+    def construct_yaml_seq(self, node: SequenceNode) -> ListNode:
+        obj, = SafeConstructor.construct_yaml_seq(self, node)  # type:ignore[no-untyped-call]
+        assert isinstance(obj, list)  # nosec
+        return ListNode(obj, node.start_mark, node.end_mark)  # nosec
+
+    def construct_yaml_null_error(self, node: Node) -> NoReturn:
+        """Throw a null error"""
+        raise CfnParseError(
+            filename=self.filename,
+            message=f"Null value at line {node.start_mark.line + 1} column {node.start_mark.column + 1}",
+            line_number=node.start_mark.line,
+            column_number=node.start_mark.column,
+        )
+
+
+class MarkedLoader(Reader, Scanner, Parser, Composer, NodeConstructor, Resolver):
+    """
+    Class for marked loading YAML
+    """
+
+    # pylint: disable=non-parent-init-called,super-init-not-called
+
+    def __init__(self, stream: str, filename: str, content_type: ContentType | None = None) -> None:
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        if cyaml:
+            Parser.__init__(self, stream)
+        else:
+            Parser.__init__(self)  # type:ignore[call-arg]  # cyaml checks if it is the normal or C version
+        Composer.__init__(self)
+        SafeConstructor.__init__(self)
+        Resolver.__init__(self)
+        NodeConstructor.__init__(self, filename, content_type)
+
+    def construct_mapping(self, node: MappingNode, deep: bool = False) -> dict[Hashable, Any]:
+        mapping = super(MarkedLoader, self).construct_mapping(node, deep=deep)
+        # Add 1 so line numbering starts at 1
+        # mapping['__line__'] = node.start_mark.line + 1
+        mapping['__startline__'] = node.start_mark.line + 1
+        mapping['__endline__'] = node.end_mark.line + 1
+        return mapping
+
+
+def multi_constructor(loader: MarkedLoader, tag_suffix: str, node: ScalarNode) -> DictNode:
+    """
+    Deal with !Ref style function format
+    """
+
+    constructor: Callable[[ScalarNode], Any]
+
+    if tag_suffix not in UNCONVERTED_SUFFIXES:
+        tag_suffix = f"{FN_PREFIX}{tag_suffix}"
+
+    if tag_suffix == 'Fn::GetAtt':
+        constructor = construct_getatt
+    elif tag_suffix == "Ref" and (isinstance(node.value, list) or isinstance(node.value, dict)):
+        raise CfnParseError(
+            filename="",
+            message='Invalid !Ref: {}'.format(node.value),
+            line_number=0,
+            column_number=0)
+    elif isinstance(node, ScalarNode):
+        constructor = loader.construct_scalar
+    elif isinstance(node, SequenceNode):
+        constructor = loader.construct_sequence
+    elif isinstance(node, MappingNode):
+        constructor = loader.construct_mapping
+    else:
+        raise Exception('Bad tag: !{}'.format(tag_suffix))
+
+    return DictNode({tag_suffix: constructor(node)}, node.start_mark, node.end_mark)
+
+
+def construct_getatt(node: ScalarNode) -> ListNode:
+    """
+    Reconstruct !GetAtt into a list
+    """
+
+    if isinstance(node.value, str):
+        return ListNode(node.value.split('.'), node.start_mark, node.end_mark)
+    if isinstance(node.value, list):
+        return ListNode([s.value for s in node.value], node.start_mark, node.end_mark)
+
+    raise ValueError('Unexpected node type: {}'.format(type(node.value)))
+
+
+def loads(yaml_string: str, fname: str, content_type: ContentType | None = None) -> DictNode | dict[str, Any]:
+    """
+    Load the given YAML string
+    """
+    loader = MarkedLoader(yaml_string, fname, content_type)
+    loader.add_multi_constructor('!', multi_constructor)  # type:ignore[no-untyped-call]
+
+    template: "DictNode | dict[str, Any]" = loader.get_single_data()
+    # Convert an empty file to an empty dict
+    if template is None:
+        template = {}
+
+    return template
+
+
+def load(filename: str | Path, content_type: ContentType) -> tuple[dict[str, Any], list[tuple[int, str]]]:
+    """
+    Load the given YAML file
+    """
+    file_path = filename if isinstance(filename, Path) else Path(filename)
+
+    if platform.system() == "Windows":
+        try:
+            content = str(from_path(file_path).best())
+        except UnicodeDecodeError as e:
+            LOGGER.error(f"Encoding for file {file_path} could not be detected or read. Please try encoding the file as UTF-8.")
+            raise e
+    else:
+        try:
+            content = file_path.read_text()
+        except UnicodeDecodeError:
+            LOGGER.info(f"Encoding for file {file_path} is not UTF-8, trying to detect it")
+            content = str(from_path(file_path).best())
+
+    if content_type == ContentType.CFN and "Resources" not in content:
+        logging.debug(f'File {file_path} is expected to be a CFN template but has no Resources attribute')
+        return {}, []
+    elif content_type == ContentType.SLS and "provider" not in content:
+        logging.debug(f'File {file_path} is expected to be an SLS template but has no provider attribute')
+        return {}, []
+    elif content_type == ContentType.TFPLAN and "planned_values" not in content:
+        logging.debug(f'File {file_path} is expected to be a TFPLAN file but has no planned_values attribute')
+        return {}, []
+
+    file_lines = [(idx + 1, line) for idx, line in enumerate(content.splitlines(keepends=True))]
+
+    if file_path.suffix == ".json":
+        file_size = len(content)
+        if file_size > MAX_IAC_FILE_SIZE:
+            # large JSON files take too much time, when parsed with `pyyaml`, compared to a normal 'json.loads()'
+            # with start/end line numbers of 0 takes only a few seconds
+            logging.info(
+                f"File {file_path} has a size of {file_size} which is bigger than the supported 50mb, "
+                "therefore file lines will default to 0."
+                "This limit can be adjusted via the environment variable 'SENTINELONE_CNS_MAX_IAC_FILE_SIZE'."
+            )
+            return json.loads(content, cls=SimpleDecoder), file_lines
+
+    return loads(content, str(filename), content_type), file_lines