conviso-ast 3.0.0__py3-none-any.whl

convisoappsec/common/docker.py

@@ -0,0 +1,399 @@
+import tarfile
+import tempfile
+from contextlib import suppress
+from datetime import datetime
+from threading import Lock, Thread
+from uuid import uuid4
+
+from convisoappsec.flow.util import SourceCodeCompressor
+from convisoappsec.logger import LOGGER
+
+import docker
+
+from .exceptions import *
+
+mutex = Lock()
+
+
+class EventLogging(Thread):
+
+    def __init__(self, logger, events):
+        super().__init__(name='Docker-Events', daemon=True)
+        self.logger = logger or LOGGER
+        self.events = events
+
+    def docker_log(self, event):
+        ts = datetime.fromtimestamp(event['time']).strftime('%d/%m/%Y %H:%M:%S')
+        message = "{timestamp}|{Type} {Action} on {Actor[ID]}".format(**event, timestamp=ts)
+        self.logger.debug(message)
+
+    def run(self):
+        for event in self.events:
+            if 'conviso-cli-' in event['Actor']['ID']:
+                self.docker_log(event)
+        self.logger.debug('Docker Events closed')
+
+
+class Credentials:
+
+    AUTHS = {}
+
+    def __init__(self, docker, logger):
+        self.docker = docker
+        self.logger = logger or LOGGER
+
+    def login(self, registry, password, username='AWS'):
+        mutex.acquire()
+        if not registry in Credentials.AUTHS.keys():
+            self.logger.debug(
+                "   \U0001F511 Checking Authorization for {0}".format(registry))
+            login_args = {
+                'registry': registry,
+                'username': username,
+                'password': password,
+                'reauth': True,
+            }
+            try:
+                login_result = self.docker.login(**login_args)
+                status = login_result['Status']
+                self.logger.debug("Login result was {0}".format(login_result))
+                if status == 'Login Succeeded':
+                    Credentials.AUTHS[registry] = password
+                    self.logger.debug('   \U0001F513 ' + status)
+            except docker.errors.APIError:
+                self.logger.debug("Docker API Error")
+        mutex.release()
+
+
+class SCSCommon:
+
+    # General Settings
+    DEFAULT_REGISTRY = 'public.ecr.aws/convisoappsec'
+    DEFAULT_TAG = 'latest'
+    DEFAULT_CONTAINER_CODE_DIR = '/code'
+    SUCCESS_EXIT_CODE = 0
+    DOCKER_CLIENT = None
+    DOCKER_CREDENTIALS = None
+    DOCKER_EVENTS = None
+    DOCKER_EVENTS_THREAD = None
+    TEMPDIR = None
+
+    def __init__(self, tag=None, registry=None, repository_name=None, token=None, logger=None, command=None, repository_dir=None):
+        self.logger = logger or LOGGER
+        self.token = token
+        uuid = str(uuid4())
+        self.docker = self._get_docker_client()
+        self.__container_name = "conviso-cli-{0}".format(uuid)
+        self.__docker_events = self._get_docker_events()
+        self.__docker_events_thread = self._get_docker_events_thread()
+        self.__credentials = self._get_docker_credentials()
+        self.__source_code_volume_name = "conviso-cli-{0}".format(uuid)
+
+        self.container = None
+        self.tag = tag or self.DEFAULT_TAG
+        self.registry = registry or self.DEFAULT_REGISTRY
+        self.repository_name = repository_name
+        self.command = command
+        self.repository_dir = repository_dir
+        if token:
+            self._login(self.token)
+
+    def _get_docker_client(self):
+        mutex.acquire()
+        if not SCSCommon.DOCKER_CLIENT:
+            SCSCommon.DOCKER_CLIENT = docker.from_env(
+                version="auto"
+            )
+        mutex.release()
+        return SCSCommon.DOCKER_CLIENT
+
+    def _get_docker_credentials(self):
+        mutex.acquire()
+        if not SCSCommon.DOCKER_CREDENTIALS:
+            SCSCommon.DOCKER_CREDENTIALS = Credentials(
+                docker=self.docker,
+                logger=self.logger
+            )
+        mutex.release()
+        return SCSCommon.DOCKER_CREDENTIALS
+
+    def _get_docker_events(self):
+        mutex.acquire()
+        if not SCSCommon.DOCKER_EVENTS:
+            SCSCommon.DOCKER_EVENTS = self.docker.events(decode=True)
+        mutex.release()
+        return SCSCommon.DOCKER_EVENTS
+
+    def _get_docker_events_thread(self):
+        mutex.acquire()
+        if not SCSCommon.DOCKER_EVENTS_THREAD:
+            SCSCommon.DOCKER_EVENTS_THREAD = EventLogging(
+                logger=self.logger,
+                events=self.__docker_events,
+            )
+            SCSCommon.DOCKER_EVENTS_THREAD.start()
+        mutex.release()
+        return SCSCommon.DOCKER_EVENTS_THREAD
+
+    def _login(self, password, username='AWS'):
+        self.__credentials.login(
+            registry=self.registry,
+            password=password,
+        )
+
+    def _create_host_output_directory(self):
+        """Creates a directory on the host machine to store container outputs.
+
+        Returns:
+            string: absolute path to the directory
+        """
+        if not SCSCommon.TEMPDIR:
+            SCSCommon.TEMPDIR = tempfile.mkdtemp(
+                prefix='conviso-output-',
+                dir=self.repository_dir
+            )
+
+        self.logger.debug("Created output directory at {}".format(
+            SCSCommon.TEMPDIR
+        ))
+        return SCSCommon.TEMPDIR
+
+    @property
+    def size(self):
+        registry_data = self.docker.images.get_registry_data(self.image)
+
+        descriptor = registry_data.attrs.get('Descriptor', {})
+        return descriptor.get('size') * 1024 * 1024
+
+    def run(self):
+        container = self.__create_container()
+        self.__inject_container_source_code(container)
+        container.start()
+
+        container_stderr = container.logs(
+            stream=True,
+            stdout=False,
+            stderr=True
+        )
+        for message in container_stderr:
+            self.logger.debug(message)
+
+        container_stdout = container.logs(
+            stream=True,
+            stdout=True,
+            stderr=False,
+        )
+
+        for message in container_stdout:
+            self.logger.debug(message)
+
+    def wait(self):
+        wait_result = self.__container.wait()
+        status_code = wait_result.get('StatusCode')
+
+        if not status_code == self.SUCCESS_EXIT_CODE:
+            raise CommonException()
+
+        return status_code
+
+    def __has_method(self, method_name):
+        return hasattr(self, method_name)
+
+    def pull(self):
+        '''
+        Acts as docker pull <image_name>, downloading the image configured in the instance.
+
+        Returns:
+            # Example:
+            {
+                'status': 'Downloading',
+                'progressDetail': {'current': int, 'total': int},
+                'id': 'string'
+            }
+        '''
+        self.logger.debug(
+            "Pulling scanner image {}. It might takes some minutes.".format(
+                self.repository_name)
+        )
+        if self.has_pre_pull:
+            self._pre_pull()
+            self.logger.debug("End of pre_pull method.")
+        try:
+            return self.docker.images.pull(
+                repository=self.repository,
+                tag=self.tag
+            )
+        except docker.errors.ImageNotFound:
+            self.logger.debug(
+                "Image {} not found on registry".format(self.image)
+            )
+            return False
+
+    def __get_container(self):
+        return self.docker.containers.get(
+            self.__container_name
+        )
+
+    @property
+    def __container(self):
+        try:
+            return self.__get_container()
+        except:
+            return self.__create_container()
+
+    def __create_container(self):
+        self.logger.debug(
+            "Creating Container {0}".format(self.__container_name)
+        )
+        return self.docker.containers.create(
+            self.image,
+            name=self.__container_name,
+            volumes=self.volumes,
+            detach=True,
+            command=self.command
+        )
+
+    @property
+    def repository(self):
+        return "{registry}/{repository_name}".format(
+            registry=self.registry,
+            repository_name=self.repository_name,
+        )
+
+    @property
+    def image(self):
+        return "{repository}:{tag}".format(
+            repository=self.repository,
+            tag=self.tag,
+        )
+
+    @property
+    def has_pre_pull(self):
+        return self.__has_method('_pre_pull')
+
+    @property
+    def has_read_scan_stdout(self):
+        return self.__has_method('_read_scan_stdout')
+
+    @property
+    def has_read_scan_stderr(self):
+        return self.__has_method('_read_scan_stderr')
+
+    @property
+    def volumes(self):
+        return {
+            self.__source_code_volume_name: {
+                'bind': SCSCommon.DEFAULT_CONTAINER_CODE_DIR,
+                'mode': 'rw',
+            }
+        }
+
+    @property
+    def name(self):
+        return self.__container_name
+
+    def __del__(self):
+        self.logger.debug(
+            "Removing container {0}".format(self.__container_name)
+        )
+        with suppress(Exception):
+            self.container.remove(v=True, force=True)
+
+    def __inject_container_source_code(self, container):
+        try:
+            self.logger.debug(
+                "Loading Source code into {0}".format(self.__container_name)
+            )
+
+            with tempfile.TemporaryFile() as wrapper_file:
+                compressor = SourceCodeCompressor(
+                    self.repository_dir
+                )
+                compressor.write_to(wrapper_file)
+                wrapper_file.seek(0)
+                container.put_archive(
+                    SCSCommon.DEFAULT_CONTAINER_CODE_DIR,
+                    wrapper_file
+                )
+
+        except Exception as e:
+            self.logger.error(e)
+
+    def _get_container_tarball_chunks(self, container_filepath):
+        """From the path provided you get the file binary/chunks from the container.
+
+        Args:
+            container_filepath (string): Absolute Path of the file within the container
+
+        Returns:
+            int: For further information see https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.Container.get_archive
+        """
+        try:
+            container = self.__container
+            chunks, _ = container.get_archive(container_filepath)
+            return chunks
+
+        except docker.errors.NotFound:
+            self.logger.debug("{} does not detected issues, continuing...".format(
+                self.repository_name
+            ))
+            return None
+
+    def _extract_tarball_chunks(self, tarball_chunks, report_filename):
+        """
+
+        Args:
+            tarball_chunks (int): The number of bytes returned by each iteration of the generator
+            report_filename (string): The name of the extracted report
+
+        Returns:
+            string: Report absolute filepath in local filesystem
+        """
+        output_dirpath = self._create_host_output_directory()
+        report_absolute_filepath = '{}/{}'.format(
+            output_dirpath, report_filename
+        )
+
+        with tempfile.TemporaryFile() as tmp_wrapper_file:
+            for chunk in tarball_chunks:
+                tmp_wrapper_file.write(chunk)
+            tmp_wrapper_file.seek(0)
+
+            with tarfile.open(mode="r|", fileobj=tmp_wrapper_file) as talball_file:
+                talball_file.extractall(path=output_dirpath)
+
+        self.logger.debug("report from {} saved to {}".format(
+            self.__container_name,
+            report_absolute_filepath
+        ))
+        return report_absolute_filepath
+
+    def get_container_reports(self):
+        """ Get all reports found in the instantiated container and save on the host machine.
+
+        Returns:
+            string: Report absolute filepath in local filesystem
+        """
+        self.logger.debug(
+            "Collecting reports from {0}".format(self.__container_name)
+        )
+        try:
+            possible_report_extensions = ['sarif', 'json']
+            for ext in possible_report_extensions:
+                report_filename = '{}.{}'.format(self.repository_name, ext)
+
+                container_filepath = '/{}'.format(report_filename)
+                tarball_chunks = self._get_container_tarball_chunks(
+                    container_filepath
+                )
+                if not tarball_chunks:
+                    continue
+
+                absolute_report_path = self._extract_tarball_chunks(
+                    tarball_chunks,
+                    report_filename
+                )
+                return absolute_report_path
+
+        except Exception as e:
+            self.logger.error(e)

convisoappsec/common/exceptions.py

@@ -0,0 +1,8 @@
+class CommonException(RuntimeError):
+    pass
+
+class ExecutionFailureException(CommonException):
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.exit_code = kwargs['exit_code']

convisoappsec/common/git_data_parser.py

@@ -0,0 +1,76 @@
+
+from git import Repo
+from git.exc import GitCommandError, InvalidGitRepositoryError
+from giturlparse import parse as gitparse
+from hashlib import sha256
+import os
+
+from convisoappsec.flow.version_control_system_adapter import GitAdapter
+from convisoappsec.logger import LOGGER
+
+
+class DescriptionParsingError(GitCommandError):
+    pass
+
+
+class GitDataParser:
+    def __init__(self, repository_dir):
+        try:
+            self.__validate_git_repo(repository_dir)
+
+            self.repository_dir = repository_dir
+            self._git_adapter = GitAdapter(repository_dir)
+
+        except InvalidGitRepositoryError as exp:
+            LOGGER.error(
+                'Invalid Git repository: "{}".'.format(repository_dir)
+            )
+            raise exp
+
+        except Exception as exp:
+            LOGGER.error("An unxpected error occurred.")
+            raise exp
+
+    def parse_name(self):
+        try:
+            git_remote_url = self._git_adapter._git_client.ls_remote(
+                '--get-url'
+            )
+
+            return self.__parse_name_from_git_url(git_remote_url)
+        except GitCommandError:
+            dirname = os.path.dirname(self.repository_dir)
+            basename = os.path.basename(self.repository_dir)
+            digest = sha256(dirname.encode('utf-8')).hexdigest()
+
+            return basename + '-' + digest
+        except Exception as exception:
+            raise exception
+
+    def __parse_description(self):
+        try:
+            readme = self._git_adapter._git_client.show(':README.md')
+            readme = readme.replace('\n', '\n<br>')
+
+            return readme
+
+        except DescriptionParsingError as exception:
+            raise exception
+
+        except Exception as exception:
+            raise exception
+
+    def __parse_name_from_git_url(self, git_url):
+        parser = gitparse(git_url)
+
+        return '{} ({}/{})'.format(
+            parser.name,
+            parser.resource,
+            parser.owner,
+        )
+
+    def __validate_git_repo(self, repository_dir):
+        repo = Repo(repository_dir)
+        repo.git.config("--global", "--add", "safe.directory", repository_dir)
+
+        return repo

convisoappsec/common/graphql/error_handlers.py

@@ -0,0 +1,75 @@
+import requests
+import jmespath
+
+from convisoappsec.common.graphql.errors import AuthenticationError, Error, ResponseError, ServerError
+
+
+class RequestErrorHandler:
+    def __init__(self, error):
+        self.error = error
+
+    def handle_request_error(self):
+        try:
+            raise self.error
+        except requests.exceptions.HTTPError as e:
+            self._handle_http_error(e)
+        except Exception as e:
+            self._handle_unexpected_error(e)
+
+    @staticmethod
+    def _handle_unexpected_error(error):
+        raise Error(str(error)) from error
+
+    @staticmethod
+    def _handle_http_error(error):
+        if error.response.status_code == 401:
+            error_msg_fmt = 'Unauthorized access, check your credentials. {}'
+            response = error.response.json()
+            error_messages = response.get('errors', [])
+            error_msg = error_msg_fmt.format(error_messages)
+
+            raise AuthenticationError(error_msg) from error
+
+        if error.response.status_code == 500:
+            error_msg_fmt = 'Internal Server Error. {}'
+            error_msg = error_msg_fmt.format(error.response.text)
+
+            raise ServerError(error_msg) from error
+
+        raise Error(error.response.text) from error
+
+
+class GraphQlErrorHandler:
+    def __init__(self, response):
+        self.response = response
+
+    def raise_on_graphql_body_error(self):
+        data = self.response.get('data', [])
+
+        for key in data:
+            if data[key] is None:
+                continue
+
+            errors = data[key].get('errors', [])
+            has_errors = len(errors) > 0
+            if has_errors:
+                raise ResponseError(errors)
+
+    def raise_on_graphql_error(self):
+        errors = self.response.get('errors', [])
+
+        if not errors:
+            return
+
+        error = errors[0]
+
+        error_path = 'extensions.code'
+        
+        code = jmespath.search(
+            error_path,
+            error
+        )
+
+        message = error.get('message', '')
+
+        raise ResponseError(message, code)

convisoappsec/common/graphql/errors.py

@@ -0,0 +1,16 @@
+class Error(Exception):
+    pass
+
+
+class AuthenticationError(Error):
+    pass
+
+
+class ServerError(Error):
+    pass
+
+
+class ResponseError(Error):
+    def __init__(self, message, code=None):
+        super().__init__(message)
+        self.code = code

convisoappsec/common/graphql/low_client.py

@@ -0,0 +1,51 @@
+import requests
+
+from convisoappsec.common.graphql.error_handlers import GraphQlErrorHandler, RequestErrorHandler
+from convisoappsec.version import __version__
+
+
+class GraphQLClient:
+    DEFAULT_HEADERS = {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+        "User-Agent": "AST:{version}".format(version=__version__),
+    }
+
+    def __init__(self, url, headers={}):
+        self.url = url
+        self.__session = requests.Session()
+        self.__session.headers.update(
+            **self.DEFAULT_HEADERS,
+            **headers
+        )
+
+    def execute(self, query, variables={}):
+        try:
+            payload = self._build_graphql_payload(query, variables)
+
+            response = self.__session.post(
+                url=self.url,
+                json=payload,
+            )
+
+            response.raise_for_status()
+
+        except Exception as e:
+            handler = RequestErrorHandler(e)
+            handler.handle_request_error()
+
+        json_response = response.json()
+        graphql_handler = GraphQlErrorHandler(json_response)
+        graphql_handler.raise_on_graphql_error()
+        graphql_handler.raise_on_graphql_body_error()
+
+        return json_response['data']
+
+    @staticmethod
+    def _build_graphql_payload(query, variables):
+        data = {
+            'query': query,
+            'variables': variables
+        }
+
+        return data

convisoappsec/common/retry_handler.py

@@ -0,0 +1,40 @@
+import time
+import traceback
+from convisoappsec.logger import LOGGER, log_and_notify_ast_event
+
+
+class RetryHandler:
+    def __init__(self, flow_context=None, company_id=None, asset_id=None):
+        self.max_retries = 5
+        self.initial_delay = 1
+        self.backoff_factor = 2
+        self.flow_context = flow_context
+        self.company_id = company_id
+        self.asset_id = asset_id
+
+    def execute_with_retry(self, func, *args, **kwargs):
+        """Execute a function with retry logic and exponential backoff."""
+        retries = 0
+        delay = self.initial_delay
+
+        while retries < self.max_retries:
+            try:
+                return func(*args, **kwargs)
+            except Exception:
+                retries += 1
+                time.sleep(delay)
+                delay *= self.backoff_factor
+
+                if retries == self.max_retries:
+                    full_trace = traceback.format_exc()
+                    LOGGER.warning(
+                        "⚠️ Maximum retries reached. Our technical team has been notified."
+                    )
+
+                    try:
+                        log_and_notify_ast_event(
+                            flow_context=self.flow_context, company_id=self.company_id, asset_id=self.asset_id,
+                            ast_log=full_trace
+                        )
+                    except Exception as log_error:
+                        LOGGER.warning(f"⚠️ Failed to log and notify AST event: {log_error}")

convisoappsec/common/strings.py

@@ -0,0 +1,8 @@
+
+def parse_to_ascii(str_value):
+    str_value_bytes = str_value.encode()
+    str_value_as_ascii = str_value_bytes.decode(
+        'ascii', 'ignore'
+    )
+
+    return str_value_as_ascii

convisoappsec/flow/__init__.py

@@ -0,0 +1,3 @@
+from .version_control_system_adapter import GitAdapter
+
+__all__ = ['GitAdapter']