stix-shifter-modules-sysdig 8.0.2__py3-none-any.whl

stix_shifter_modules/sysdig/stix_translation/query_constructor.py

@@ -0,0 +1,472 @@
+""" This Module will convert Stix Pattern to Sysdig data source supported query """
+import re
+import json
+import logging
+from os import path
+from datetime import datetime, timedelta
+from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \
+    ComparisonExpression, ComparisonComparators, Pattern, \
+    CombinedComparisonExpression, CombinedObservationExpression
+
+logger = logging.getLogger(__name__)
+
+START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)"
+MAC = '^(([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2}))$'
+TYPE_MAP_PATH = "json/config_map.json"
+
+STOP_TIME = datetime.utcnow()
+
+
+class StartStopQualifierValueException(Exception):
+    """ Start Stop qualifier exception """
+    pass
+
+
+class FileNotFoundException(Exception):
+    """ file not found exception """
+    pass
+
+
+class QueryStringPatternTranslator:
+    """
+    translate stix pattern to native data source query language
+    """
+
+    def __init__(self, pattern: Pattern, data_model_mapper, options):
+        logger.info("Sysdig Connector")
+        self.dmm = data_model_mapper
+        self.options = options
+        self.comparator_lookup = self.dmm.map_comparator()
+        self.type_map = self.load_json(TYPE_MAP_PATH)
+        self.translated_query = self.parse_expression(pattern)
+
+    @staticmethod
+    def load_json(rel_path_of_file):
+        """
+        Consumes a json file and returns a dictionary
+        :param rel_path_of_file: str
+        :return: dict
+        """
+        _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file
+        try:
+            if path.exists(_json_path):
+                with open(_json_path, encoding='utf-8') as f_obj:
+                    return json.load(f_obj)
+            raise FileNotFoundException
+        except FileNotFoundException as ex:
+            raise FileNotFoundError(f'{rel_path_of_file} not found') from ex
+
+    def _format_set(self, values, mapped_field_type, mapped_fields_array):
+        """
+        Formats value in the event of set operation
+        :param values
+        :param mapped_field_type: str
+        :param mapped_fields_array: list
+        :return formatted value
+        """
+        gen = values.element_iterator()
+        formatted_value = ','.join(QueryStringPatternTranslator._escape_value(
+            self._format_value_type(value, mapped_field_type, mapped_fields_array), mapped_field_type)
+                                   for value in gen)
+        return f'({formatted_value})'
+
+    @staticmethod
+    def _format_equality(value, mapped_field_type):
+        """
+        Formats value in the event of equality operation
+        :param value
+        :return formatted value
+        """
+        return QueryStringPatternTranslator._escape_value(value, mapped_field_type)
+
+    @staticmethod
+    def _escape_value(value, mapped_field_type):
+        """
+        adds escape characters to string type value
+        :param value
+        :return formatted value
+        """
+        if mapped_field_type == "int":
+            return value
+        if isinstance(value, str):
+            value = f'\"{value}\"'
+        return str(value)
+
+    @staticmethod
+    def _field_severity(value):
+        """
+        convert severity 1-100 to data source value 0-7
+        High: 0-3, Medium: 4-5, Low: 6, Info: 7
+        param value
+        return value(int)
+        examples:
+            severity is 100, after conversion is 0.
+            severity is 30, after conversion is 7.
+        Reference :
+        Sysdig Docs https://docs.sysdig.com/en/docs/sysdig-secure/secure-events/event-forwarding/#policy-event-severity
+        """
+        value = int(value)
+        if 91 <= value <= 100:
+            value = 0
+        elif 81 <= value <= 90:
+            value = 1
+        elif 71 <= value <= 80:
+            value = 2
+        elif 61 <= value <= 70:
+            value = 3
+        elif 51 <= value <= 60:
+            value = 4
+        elif 41 <= value <= 50:
+            value = 5
+        elif 31 <= value <= 40:
+            value = 6
+        elif 1 <= value <= 30:
+            value = 7
+        else:
+            raise NotImplementedError('only 1-100 integer values are supported with this field')
+        return value
+
+    def _format_value_type(self, value, mapped_field_type, mapped_fields_array):
+        """
+        check input value format that matches with the mapped field value type
+        :param value
+        :param mapped_field_type: str
+        :param mapped_fields_array: list
+        :return formatted value
+        """
+        converted_value = str(value)
+        if mapped_field_type == "mac":
+            compile_mac_regex = re.compile(MAC)
+            if not compile_mac_regex.search(converted_value):
+                raise NotImplementedError(f'Invalid mac address - {converted_value} provided')
+        elif mapped_field_type == "int":
+            if not converted_value.isdigit():
+                raise NotImplementedError(f'string type input - {converted_value} is not supported for '
+                                          f'integer type fields')
+            if mapped_fields_array[0] == "severity":
+                value = self._field_severity(value)
+            converted_value = str(value)
+        return converted_value
+
+    def _check_value_comparator_support(self, comparator, mapped_field_type):
+        """
+        checks the comparator and value support
+        :param comparator
+        :param mapped_field_type: str
+        """
+        operator = self.comparator_lookup[str(comparator)]
+
+        if mapped_field_type == 'string' and comparator not in [ComparisonComparators.In,
+                                                                ComparisonComparators.Equal,
+                                                                ComparisonComparators.NotEqual
+                                                                ]:
+            raise NotImplementedError(f'{operator} operator is not supported for string type input')
+        if mapped_field_type == 'int' and comparator not in [ComparisonComparators.In,
+                                                             ComparisonComparators.Equal,
+                                                             ComparisonComparators.NotEqual,
+                                                             ComparisonComparators.GreaterThan,
+                                                             ComparisonComparators.GreaterThanOrEqual,
+                                                             ComparisonComparators.LessThanOrEqual,
+                                                             ComparisonComparators.LessThan]:
+            raise NotImplementedError(f'{operator} operator is not supported for integer type input')
+
+    @staticmethod
+    def _format_negate(comparator):
+        """
+        returns negation of input operator
+        :param comparator:str
+        :return str
+        """
+        negate_comparator = {
+            ">": "<=",
+            ">=": "<",
+            "<": ">=",
+            "<=": ">",
+            "=": "!=",
+            "in": "not in"
+        }
+        return negate_comparator[comparator]
+
+    @staticmethod
+    def _check_time_range_values(from_epoch, to_epoch):
+        """
+        checks for valid start and stop time
+        :param from_epoch: int
+        :param to_epoch: int
+        """
+        if from_epoch > to_epoch:
+            raise StartStopQualifierValueException('Start time should be lesser than Stop time')
+
+        no_of_days = (to_epoch - from_epoch)/86400000000000       # converting nano seconds to days
+        max_day = 14                                              # time range maximum up to 14 days
+        if max_day < no_of_days:
+            logger.warning('Sysdig connector query time range can be maximum up to 14 days')
+
+    @staticmethod
+    def _parse_time_range(qualifier, time_range):
+        """
+        Converts qualifier timestamp to epoch
+        :param qualifier: str
+        :param time_range: int
+        return: list of converted epoch values
+        """
+        try:
+            compile_timestamp_regex = re.compile(START_STOP_PATTERN)
+            if qualifier and compile_timestamp_regex.search(qualifier):
+                time_range_iterator = compile_timestamp_regex.finditer(qualifier)
+                time_range_list = [each.group() for each in time_range_iterator]
+            else:
+                start_time = STOP_TIME - timedelta(minutes=time_range)
+                converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+                # limit 3 digit value for millisecond
+                converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+                time_range_list = [converted_start_time, converted_stop_time]
+        except (KeyError, IndexError, TypeError) as ex:
+            raise ex
+        return time_range_list
+
+    @staticmethod
+    def get_epoch_time(timestamp):
+        """
+        Converting timestamp (YYYY-MM-DDThh:mm:ss.000Z) to 13-digit Unix time (epoch + milliseconds)
+        :param timestamp: str, timestamp
+        :return: int, epoch time
+        """
+        time_patterns = ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S.%fZ']
+        epoch = datetime(1970, 1, 1)
+        for time_pattern in time_patterns:
+            try:
+                converted_time = int(
+                    ((datetime.strptime(timestamp, time_pattern) - epoch).total_seconds()) * 1000 * 1000000)
+                return converted_time
+            except ValueError:
+                pass
+        raise NotImplementedError("cannot convert the timestamp {} to epoch time".format(timestamp))
+
+    def _add_timestamp_to_query(self, query, qualifier):
+        """
+        adds timestamp filter to Sysdig query
+        :param query: str
+        :param qualifier
+        :return str
+        """
+        converted_timestamp = \
+            QueryStringPatternTranslator._parse_time_range(qualifier, self.options["time_range"])
+        from_epoch = self.get_epoch_time(converted_timestamp[0])
+        to_epoch = self.get_epoch_time(converted_timestamp[1])
+        QueryStringPatternTranslator._check_time_range_values(from_epoch, to_epoch)
+        final_query = f'from={from_epoch}&to={to_epoch}&filter={query}'
+        return final_query
+
+    def _check_mapped_field_type(self, mapped_field_array):
+        """
+        Returns the type of mapped field array
+        :param mapped_field_array: list
+        :return: str
+        """
+        mapped_field = mapped_field_array[0]
+        mapped_field_type = "string"
+        for key, value in self.type_map.items():
+            if mapped_field in value and key in ["int_supported_fields", "string_supported_fields",
+                                                 "mac_supported_fields"]:
+                mapped_field_type = key.split('_')[0]
+                break
+        return mapped_field_type
+
+    @staticmethod
+    def _parse_mapped_fields(value, comparator, mapped_fields_array):
+        """
+        parse mapped fields into query expression
+        :param value: str
+        :param comparator: str
+        :param mapped_fields_array: list
+        :return: str
+        """
+        comparison_string = ""
+        mapped_fields_count = len(mapped_fields_array)
+        for mapped_field in mapped_fields_array:
+            if "in" in comparator:
+                comparison_string += f'{mapped_field} {comparator} {value}'
+            else:
+                comparison_string += f'{mapped_field}{comparator}{value}'
+            if mapped_fields_count > 1:
+                comparison_string += " OR "
+                mapped_fields_count -= 1
+        return comparison_string
+
+    def _lookup_comparison_operator(self, expression_operator):
+        """
+        lookup operators support in Sysdig connector
+        :param expression_operator:enum object
+        :return str
+        """
+        if str(expression_operator) not in self.comparator_lookup:
+            raise NotImplementedError(
+                f'Comparison operator {expression_operator.name} unsupported for Sysdig connector')
+        operator = self.comparator_lookup[str(expression_operator)]
+        return operator
+
+    def _eval_comparison_value(self, expression, mapped_field_type, mapped_fields_array):
+        """
+        Function for parsing comparison expression value
+        :param expression: expression object
+        :param mapped_field_type:str
+        :param mapped_fields_array: list
+        :return: formatted expression value
+        """
+        if expression.comparator == ComparisonComparators.In:
+            value = self._format_set(expression.value, mapped_field_type, mapped_fields_array)
+        elif expression.comparator not in [ComparisonComparators.Like, ComparisonComparators.Matches]:
+            value = self._format_value_type(expression.value, mapped_field_type, mapped_fields_array)
+            self._check_value_comparator_support(expression.comparator, mapped_field_type)
+            value = self._format_equality(value, mapped_field_type)
+        else:
+            raise NotImplementedError('Unknown comparator expression operator')
+        return value
+
+    def _eval_combined_comparison_exp(self, expression):
+        """
+        Function for parsing combined comparison expression
+        :param expression: expression object
+        """
+        operator = self._lookup_comparison_operator(expression.operator)
+        expression_01 = self._parse_expression(expression.expr1)
+        expression_02 = self._parse_expression(expression.expr2)
+        if not expression_01 or not expression_02:
+            return ''
+        if isinstance(expression.expr1, CombinedComparisonExpression):
+            expression_01 = f'{expression_01}'
+        if isinstance(expression.expr2, CombinedComparisonExpression):
+            expression_02 = f'{expression_02}'
+
+        query_string = f'{expression_01}{operator}{expression_02}'
+        return f'{query_string}'
+
+    def _eval_combined_observation_exp(self, expression, qualifier=None):
+        """
+        Function for parsing combined observation expression
+        :param expression: expression object
+        :param qualifier: qualifier
+        """
+        operator = self._lookup_comparison_operator(expression.operator)
+        expression_01 = self._parse_expression(expression.expr1, qualifier)
+        expression_02 = self._parse_expression(expression.expr2, qualifier)
+        query = ''
+        if expression_01 and expression_02:
+            query = f'{expression_01} {operator} {expression_02}'
+
+        elif expression_01:
+            query = f'{expression_01}'
+        elif expression_02:
+            query = f'{expression_02}'
+        return query
+
+    def _parse_expression(self, expression, qualifier=None):
+        """
+        parse ANTLR pattern to Sysdig native query
+        :param expression: expression object, ANTLR parsed expression object
+        :param qualifier: str, default in None
+        :return str
+        """
+        if isinstance(expression, ComparisonExpression):  # Base Case
+            stix_objects = expression.object_path.split(':')
+            mapped_fields_array = self.dmm.map_field(stix_objects[0], stix_objects[1])
+            comparator = self._lookup_comparison_operator(expression.comparator)
+            if expression.negated:
+                comparator = QueryStringPatternTranslator._format_negate(comparator)
+            mapped_field_type = self._check_mapped_field_type(mapped_fields_array)
+            value = self._eval_comparison_value(expression, mapped_field_type, mapped_fields_array)
+            comparison_string = self._parse_mapped_fields(value, comparator, mapped_fields_array)
+            return comparison_string
+
+        elif isinstance(expression, CombinedComparisonExpression):
+            return self._eval_combined_comparison_exp(expression)
+
+        elif isinstance(expression, ObservationExpression):
+            query_string = self._parse_expression(expression.comparison_expression)
+            query_string = self._add_timestamp_to_query(query_string, qualifier)
+            return query_string
+
+        elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'):
+            if isinstance(expression.observation_expression, CombinedObservationExpression):
+                operator = self._lookup_comparison_operator(expression.observation_expression.operator)
+                expression_01 = self._parse_expression(expression.observation_expression.expr1,
+                                                       expression.qualifier)
+                expression_02 = self._parse_expression(expression.observation_expression.expr2,
+                                                       expression.qualifier)
+                query_string = f'{expression_01} {operator} {expression_02}'
+            else:
+                query_string = self._parse_expression(expression.observation_expression,
+                                                      expression.qualifier)
+            if qualifier is not None:
+                query_string = self._add_timestamp_to_query(query_string, qualifier)
+            return query_string
+
+        elif isinstance(expression, CombinedObservationExpression):
+
+            return self._eval_combined_observation_exp(expression, qualifier)
+
+        elif isinstance(expression, Pattern):
+            return self._parse_expression(expression.expression)
+        else:
+            raise RuntimeError(f'Unknown Recursion Case for expression={expression},'
+                               f' type(expression)={type(expression)}')
+
+    def parse_expression(self, pattern: Pattern):
+        """
+        Conversion of ANTLR pattern to Sysdig query
+        :param pattern: expression object, ANTLR parsed expression object
+        :return: str, native query
+        """
+        return self._parse_expression(pattern)
+
+    @staticmethod
+    def format_multiple_observations(query):
+        """
+        timestamp formatting for multiple observations
+        """
+        split_queries = query.split(' or ')
+        query_dict = {}
+        for row in split_queries:
+            # making single time stamp instead of having multiple same time stamps for multiple queries
+            split_query = row.split('filter=')
+            query_timestamp = split_query[0]
+            if query_dict.get(query_timestamp):
+                query_dict[query_timestamp] += " or " + split_query[1]
+            else:
+                query_dict[query_timestamp] = 'filter=' + split_query[1]
+        final_query = [key + value for key, value in query_dict.items()]
+        return final_query
+
+    @staticmethod
+    def removing_audit_trail_logs(format_queries):
+        """
+        Displays error message when searched for auditTrail events.
+        Adds the condition to filter auditTrail events for other queries.
+        """
+        query_list = []
+        remove_audit = "andsource!=\"auditTrail\""
+        if any('auditTrail' in query for query in format_queries):
+            raise NotImplementedError('Sysdig connector does not provide auditTrail event')
+        else:
+            for query in format_queries:
+                split_format_query = query.split('filter=')
+                query_list.append(f'{split_format_query[0]}filter=({split_format_query[1]}){remove_audit}')
+        return query_list
+
+def translate_pattern(pattern: Pattern, data_model_mapping, options):
+    """
+    Conversion of ANTLR pattern to native data source query
+    :param pattern: expression object, ANTLR parsed expression object
+    :param data_model_mapping: DataMapper object, mapping object obtained by parsing json
+    :param options: dict, time_range defaults to 5
+    :return: string, Sysdig  queries
+    """
+    translated_query_strings = QueryStringPatternTranslator(pattern, data_model_mapping, options)
+    final_queries = translated_query_strings.translated_query
+
+    if final_queries.count('from') > 1:
+        final_queries = QueryStringPatternTranslator.format_multiple_observations(final_queries)
+        return QueryStringPatternTranslator.removing_audit_trail_logs(final_queries)
+    else:
+        return QueryStringPatternTranslator.removing_audit_trail_logs([final_queries])

stix_shifter_modules/sysdig/stix_translation/query_translator.py

@@ -0,0 +1,26 @@
+import logging
+
+from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator
+from . import query_constructor
+
+logger = logging.getLogger(__name__)
+
+
+class QueryTranslator(BaseQueryTranslator):
+
+    def transform_antlr(self, data, antlr_parsing_object):
+        """
+        Transforms STIX pattern into a different query format. Based on a mapping file
+        :param antlr_parsing_object: Antlr parsing objects for the STIX pattern
+        :type antlr_parsing_object: object
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into another format. This should default to something if one isn't passed in
+        :type mapping: str (filepath)
+        :return: transformed query string
+        :rtype: str
+        """
+
+        logger.info("Converting STIX2 Pattern to data source query")
+
+        query_string = query_constructor.translate_pattern(
+            antlr_parsing_object, self, self.options)
+        return query_string

stix_shifter_modules/sysdig/stix_translation/transformers.py

@@ -0,0 +1,66 @@
+from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer
+from stix_shifter_utils.utils import logger
+import re
+from datetime import datetime
+
+LOGGER = logger.set_logger(__name__)
+connector = __name__.split('.')[1]
+
+
+class TimestampConversion(ValueTransformer):
+    """ Convert timezone date to timestamp (YYYY-MM-DDThh:mm:ss.000Z)"""
+
+    @staticmethod
+    def transform(timestamp):
+        try:
+            # with milliseconds
+            if re.search(r"\d{4}(-\d{2}){2} \d{2}(:\d{2}){2}.\d{0,6}\+\d{2}:\d{2}", str(timestamp)):
+                converted_date = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f%z")
+                timestamp = datetime.strftime(converted_date, "%Y-%m-%dT%H:%M:%S.%fZ")
+            # for without milliseconds, setting three zeroes in the millisecond in the converted date
+            elif re.search(r"\d{4}(-\d{2}){2} \d{2}(:\d{2}){2}\+\d{2}:\d{2}", str(timestamp)):
+                converted_date = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S%z")
+                timestamp = datetime.strftime(converted_date, "%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z'
+        except:
+            LOGGER.error(f'{connector} connector error cannot convert this timestamp %s', timestamp)
+        return timestamp
+
+
+class SeverityToScore(ValueTransformer):
+
+    @staticmethod
+    def transform(severity):
+        """
+        sysdig severity values levels 0-7 :
+        High: 0-3, Medium: 4-5, Low: 6, Info: 7
+        converting sysdig severity values 0-7 to 1-100
+        Example :
+            High: sysdig severity : 0, after conversion : 100
+            Low : sysdig severity : 6, after conversion : 40
+        Reference :
+        Sysdig Docs https://docs.sysdig.com/en/docs/sysdig-secure/secure-events/event-forwarding/#policy-event-severity
+        """
+        try:
+            # value transformer to convert severity value on a scale of 1-100
+            if isinstance(severity, int):
+                return (10 - int(severity)) * 10
+            return severity
+        except KeyError:
+            LOGGER.error(f'{connector} connector error cannot convert severity scale value')
+
+
+class HostnameToIpAddress(ValueTransformer):
+    """
+    Converts Node Name into IP address
+    """
+    @staticmethod
+    def transform(address):
+        try:
+            match = re.search(r'\d+\-\d+\-\d+\-\d+', address)
+            if match:
+                # Getting the matched IP address
+                extracted_ip = match.group()
+                return extracted_ip.replace('-', '.')
+            return address
+        except KeyError:
+            LOGGER.error(f'{connector} connector error cannot convert ip value')

stix_shifter_modules/sysdig/stix_transmission/api_client.py

@@ -0,0 +1,37 @@
+from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync
+
+
+class APIClient:
+    PING_ENDPOINT = 'api/v1/secureEvents/status'
+    EVENTS_ENDPOINT = 'api/v1/secureEvents?'
+
+    def __init__(self, connection, configuration):
+        self.auth = configuration.get('auth')
+        self.result_limit = connection['options'].get("result_limit")
+        headers = {}
+        if 'token' in self.auth:
+            headers['Authorization'] = "Bearer " + self.auth.get('token')
+        headers["Accept"] = "application/json"
+        headers["Content-Type"] = "application/json;charset=UTF-8"
+        # Added self-signed certificate parameter for verification
+        self.client = RestApiClientAsync(connection.get('host'),
+                                         connection.get('port', None),
+                                         headers,
+                                         url_modifier_function=None,
+                                         cert_verify=connection.get('selfSignedCert'))
+        self.timeout = connection['options'].get('timeout')
+
+    async def ping_data_source(self):
+        """
+        ping the Data Source
+        :return: Response object
+        """
+        return await self.client.call_api(self.PING_ENDPOINT, 'GET', headers=self.client.headers, timeout=self.timeout)
+
+    async def get_search_results(self, query):
+        """
+        :param query: Data Source Query
+        :return: Response Object
+        """
+        return await self.client.call_api(self.EVENTS_ENDPOINT, 'GET', urldata=query,
+                                          headers=self.client.headers, timeout=self.timeout)