misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/import_mod/cof2misp.py

@@ -0,0 +1,264 @@
+""" PassiveDNS Common Output Format (COF) MISP importer.
+
+Takes as input a valid COF file or the output of the dnsdbflex utility
+and creates MISP objects for the input.
+
+Copyright 2021: Farsight Security (https://www.farsightsecurity.com/)
+
+Author: Aaron Kaplan <aaron@lo-res.org>
+
+Released under the Apache 2.0 license.
+See: https://www.apache.org/licenses/LICENSE-2.0.txt
+
+"""
+
+import sys
+import json
+import base64
+
+
+import ndjson
+
+# from pymisp import MISPObject, MISPEvent, PyMISP
+from pymisp import MISPObject
+
+from cof2misp.cof import validate_cof, validate_dnsdbflex
+
+
+create_specific_attributes = False          # this is for https://github.com/MISP/misp-objects/pull/314
+
+
+misperrors = {'error': 'Error'}
+userConfig = {}
+
+inputSource = ['file']
+
+mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'],
+                  'format': 'misp_standard'}
+
+
+moduleinfo = {
+    'version': '0.3',
+    'author': 'Aaron Kaplan',
+    'description': 'Passive DNS Common Output Format (COF) MISP importer',
+    'module-type': ['import'],
+    'name': 'PDNS COF Importer',
+    'requirements': ['PyMISP'],
+    'features': 'Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input.',
+    'references': ['https://tools.ietf.org/id/draft-dulaunoy-dnsop-passive-dns-cof-08.html'],
+    'input': 'Passive DNS output in Common Output Format (COF)',
+    'output': 'MISP objects',
+    'logo': '',
+}
+
+moduleconfig = []
+
+
+# misp = PyMISP()
+
+
+def parse_and_insert_cof(data: str) -> dict:
+    """Parse and validate the COF data.
+
+    Parameters
+    ----------
+      data as a string
+
+    Returns
+    -------
+      A dict with either the error message or the data which may be sent off the the caller of handler()
+
+    Raises
+    --------
+      none. All Exceptions will be handled here. On error, a misperror is returned.
+    """
+
+    objects = []
+    try:
+        entries = ndjson.loads(data)
+        for entry in entries:           # iterate over all ndjson lines
+
+            # validate here (simple validation or full JSON Schema validation)
+            if not validate_cof(entry):
+                return {"error": "Could not validate the COF input '%s'" % entry}
+
+            # Next, extract some fields
+            rrtype = entry['rrtype'].upper()
+            rrname = entry['rrname'].rstrip('.')
+            rdata = [x.rstrip('.') for x in entry['rdata']]
+
+            # create a new MISP object, based on the passive-dns object for each nd-JSON line
+            o = MISPObject(name='passive-dns', standalone=False, comment='created by cof2misp')
+
+            # o.add_tag('tlp:amber')                                    # FIXME: we'll want to add a tlp: tag to the object
+            if 'bailiwick' in entry:
+                o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.'), distribution=0)
+
+            #
+            # handle the combinations of rrtype (domain, ip) on both left and right side
+            #
+
+            if create_specific_attributes:
+                if rrtype in ['A', 'AAAA', 'A6']:                           # address type
+                    # address type
+                    o.add_attribute('rrname_domain', value=rrname, distribution=0)
+                    for r in rdata:
+                        o.add_attribute('rdata_ip', value=r, distribution=0)
+                elif rrtype in ['CNAME', 'DNAME', 'NS']:                    # both sides are domains
+                    o.add_attribute('rrname_domain', value=rrname, distribution=0)
+                    for r in rdata:
+                        o.add_attribute('rdata_domain', value=r, distribution=0)
+                elif rrtype in ['SOA']:                                     # left side is a domain, right side is text
+                    o.add_attribute('rrname_domain', value=rrname, distribution=0)
+
+            #
+            # now do the regular filling up of rrname, rrtype, time_first, etc.
+            #
+            o.add_attribute('rrname', value=rrname, distribution=0)
+            o.add_attribute('rrtype', value=rrtype, distribution=0)
+            for r in rdata:
+                o.add_attribute('rdata', value=r, distribution=0)
+            o.add_attribute('raw_rdata', value=json.dumps(rdata), distribution=0)       # FIXME: do we need to hex encode it?
+            o.add_attribute('time_first', value=entry['time_first'], distribution=0)
+            o.add_attribute('time_last', value=entry['time_last'], distribution=0)
+            o.first_seen = entry['time_first']      # is this redundant?
+            o.last_seen = entry['time_last']
+
+            #
+            # Now add the other optional values.                        # FIXME: how about a map() other function. DNRY
+            #
+            for k in ['count', 'sensor_id', 'origin', 'text', 'time_first_ms', 'time_last_ms', 'zone_time_first', 'zone_time_last']:
+                if k in entry and entry[k]:
+                    o.add_attribute(k, value=entry[k], distribution=0)
+
+            #
+            # add COF entry to MISP object
+            #
+            objects.append(o.to_json())
+
+        r = {'results': {'Object': [json.loads(o) for o in objects]}}
+    except Exception as ex:
+        misperrors["error"] = "An error occured during parsing of input: '%s'" % (str(ex),)
+        return misperrors
+    return r
+
+
+def parse_and_insert_dnsdbflex(data: str):
+    """Parse and validate the more simplier dndsdbflex output data.
+
+    Parameters
+    ----------
+      data as a string
+
+    Returns
+    -------
+      A dict with either the error message or the data which may be sent off the the caller of handler()
+
+    Raises
+    --------
+      none
+    """
+    objects = []
+    try:
+        entries = ndjson.loads(data)
+        for entry in entries:           # iterate over all ndjson lines
+            # validate here (simple validation or full JSON Schema validation)
+            if not validate_dnsdbflex(entry):
+                return {"error": "Could not validate the dnsdbflex input '%s'" % entry}
+
+            # Next, extract some fields
+            rrtype = entry['rrtype'].upper()
+            rrname = entry['rrname'].rstrip('.')
+
+            # create a new MISP object, based on the passive-dns object for each nd-JSON line
+            try:
+                o = MISPObject(name='passive-dns', standalone=False, distribution=0, comment='DNSDBFLEX import by cof2misp')
+                o.add_attribute('rrtype', value=rrtype, distribution=0, comment='DNSDBFLEX import by cof2misp')
+                o.add_attribute('rrname', value=rrname, distribution=0, comment='DNSDBFLEX import by cof2misp')
+            except Exception as ex:
+                print("could not create object. Reason: %s" % str(ex))
+
+            #
+            # add dnsdbflex entry to MISP object
+            #
+            objects.append(o.to_json())
+
+        r = {'results': {'Object': [json.loads(o) for o in objects]}}
+    except Exception as ex:
+        misperrors["error"] = "An error occured during parsing of input: '%s'" % (str(ex),)
+        return misperrors
+    return r
+
+
+def is_dnsdbflex(data: str) -> bool:
+    """Check if the supplied data conforms to the dnsdbflex output (which only contains rrname and rrtype)
+
+    Parameters
+    ----------
+      ndjson data as a string
+
+    Returns
+    -------
+      True or False
+
+    Raises
+    --------
+      none
+    """
+
+    try:
+        j = ndjson.loads(data)
+        for line in j:
+            if not set(line.keys()) == {'rrname', 'rrtype'}:
+                return False            # shortcut. We assume it's not if a single line does not conform
+        return True
+    except Exception as ex:
+        print("oops, this should not have happened. Maybe not an ndjson file? Reason: %s" % (str(ex),), file=sys.stderr)
+        return False
+
+
+def is_cof(data: str) -> bool:
+    return True
+
+
+def handler(q=False):
+    if q is False:
+        return False
+
+    request = json.loads(q)
+    # Parse the json, determine which type of JSON it is (dnsdbflex or COF?)
+    # Validate it
+    # transform into MISP object
+    # push to MISP
+    # event_id = request['event_id']
+    # event = misp.get_event(event_id)
+    # print("event_id = %s" % event_id, file=sys.stderr)
+    try:
+        data = base64.b64decode(request["data"]).decode('utf-8')
+        if not data:
+            return json.dumps({'success': 0})       # empty file is ok
+        if is_dnsdbflex(data):
+            return parse_and_insert_dnsdbflex(data)
+        elif is_cof(data):
+            # check if it's valid COF format
+            return parse_and_insert_cof(data)
+        else:
+            return {'error': 'Could not find any valid COF input nor dnsdbflex input. Please have a loot at: https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/'}
+    except Exception as ex:
+        print("oops, got exception %s" % str(ex), file=sys.stderr)
+        return {'error': "Got exception %s" % str(ex)}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
+
+
+if __name__ == '__main__':
+    x = open('test.json', 'r')
+    r = handler(q=x.read())
+    print(json.dumps(r))

misp_modules/modules/import_mod/csvimport.py

@@ -0,0 +1,319 @@
+from pymisp import MISPEvent, MISPObject
+import csv
+import io
+import base64
+
+misperrors = {'error': 'Error'}
+moduleinfo = {
+    'version': '0.2',
+    'author': 'Christian Studer',
+    'module-type': ['import'],
+    'name': 'CSV Import',
+    'description': 'Module to import MISP attributes from a csv file.',
+    'requirements': ['PyMISP'],
+    'features': "In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types.",
+    'references': ['https://tools.ietf.org/html/rfc4180', 'https://tools.ietf.org/html/rfc7111'],
+    'input': 'CSV format file.',
+    'output': 'MISP Event attributes',
+    'logo': '',
+}
+moduleconfig = []
+userConfig = {
+    'header': {
+        'type': 'String',
+        'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'},
+    'has_header': {
+        'type': 'Boolean',
+        'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'},
+    'special_delimiter': {
+        'type': 'String',
+        'message': 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t" for tabs, etc).'
+    }
+}
+mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'}
+
+misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date',
+                            'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category']
+misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution',
+                                  'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag']
+misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields
+
+
+class CsvParser:
+    def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes, categories):
+        self.misp_event = MISPEvent()
+        self.header = header
+        self.has_header = has_header
+        self.delimiter = delimiter
+        self.data = data
+        self.from_misp = from_misp
+        self.MISPtypes = MISPtypes
+        self.categories = categories
+        self.fields_number = len(self.header)
+        self.__score_mapping = {0: self.__create_standard_attribute,
+                                1: self.__create_attribute_with_ids,
+                                2: self.__create_attribute_with_tags,
+                                3: self.__create_attribute_with_ids_and_tags,
+                                4: self.__create_attribute_check_category,
+                                5: self.__create_attribute_check_category_and_ids,
+                                6: self.__create_attribute_check_category_and_tags,
+                                7: self.__create_attribute_check_category_with_ids_and_tags}
+
+    def parse_csv(self):
+        if self.from_misp:
+            if self.header == misp_standard_csv_header:
+                self.__parse_misp_csv()
+            else:
+                attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:10]
+                object_fields = ['object_id'] + misp_standard_csv_header[10:]
+                attribute_indexes = []
+                object_indexes = []
+                for i in range(len(self.header)):
+                    if self.header[i] in attribute_fields:
+                        attribute_indexes.append(i)
+                    elif self.header[i] in object_fields:
+                        object_indexes.append(i)
+                if object_indexes:
+                    if not any(field in self.header for field in ('object_uuid', 'object_id')) or 'object_name' not in self.header:
+                        for line in self.data:
+                            for index in object_indexes:
+                                if line[index].strip():
+                                    return {'error': 'It is not possible to import MISP objects from your csv file if you do not specify any object identifier and object name to separate each object from each other.'}
+                    if 'object_relation' not in self.header:
+                        return {'error': 'In order to import MISP objects, an object relation for each attribute contained in an object is required.'}
+                self.__build_misp_event(attribute_indexes, object_indexes)
+        else:
+            attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:9]
+            attribute_indexes = []
+            types_indexes = []
+            for i in range(len(self.header)):
+                if self.header[i] in attribute_fields:
+                    attribute_indexes.append(i)
+                elif self.header[i] in self.MISPtypes:
+                    types_indexes.append(i)
+            self.__parse_external_csv(attribute_indexes, types_indexes)
+        self.__finalize_results()
+        return {'success': 1}
+
+    ################################################################################
+    #                      Parsing csv data with MISP fields,                      #
+    #                             but a custom header                              #
+    ################################################################################
+
+    def __build_misp_event(self, attribute_indexes, object_indexes):
+        score = self.__get_score()
+        if object_indexes:
+            objects = {}
+            id_name = 'object_id' if 'object_id' in self.header else 'object_uuid'
+            object_id_index = self.header.index(id_name)
+            name_index = self.header.index('object_name')
+            for line in self.data:
+                attribute = self.__score_mapping[score](line, attribute_indexes)
+                object_id = line[object_id_index]
+                if object_id:
+                    if object_id not in objects:
+                        misp_object = MISPObject(line[name_index])
+                        if id_name == 'object_uuid':
+                            misp_object.uuid = object_id
+                        objects[object_id] = misp_object
+                    objects[object_id].add_attribute(**attribute)
+                else:
+                    self.event.add_attribute(**attribute)
+            for misp_object in objects.values():
+                self.misp_event.add_object(**misp_object)
+        else:
+            for line in self.data:
+                attribute = self.__score_mapping[score](line, attribute_indexes)
+                self.misp_event.add_attribute(**attribute)
+
+    ################################################################################
+    #               Parsing csv data containing fields that are not                #
+    #                  MISP attributes or objects standard fields                  #
+    #                    (but should be MISP attribute types!!)                    #
+    ################################################################################
+
+    def __parse_external_csv(self, attribute_indexes, types_indexes):
+        score = self.__get_score()
+        if attribute_indexes:
+            for line in self.data:
+                try:
+                    base_attribute = self.__score_mapping[score](line, attribute_indexes)
+                except IndexError:
+                    continue
+                for index in types_indexes:
+                    attribute = {'type': self.header[index], 'value': line[index]}
+                    attribute.update(base_attribute)
+                    self.misp_event.add_attribute(**attribute)
+        else:
+            for line in self.data:
+                for index in types_indexes:
+                    self.misp_event.add_attribute(**{'type': self.header[index], 'value': line[index]})
+
+    ################################################################################
+    #                       Parsing standard MISP csv format                       #
+    ################################################################################
+
+    def __parse_misp_csv(self):
+        objects = {}
+        attribute_fields = self.header[:1] + self.header[2:8]
+        for line in self.data:
+            a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number]
+            attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))}
+            attribute['to_ids'] = True if attribute['to_ids'] == '1' else False
+            if tag:
+                attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')]
+            if relation:
+                if o_uuid not in objects:
+                    objects[o_uuid] = MISPObject(name)
+                objects[o_uuid].add_attribute(relation, **attribute)
+            else:
+                self.misp_event.add_attribute(**attribute)
+        for uuid, misp_object in objects.items():
+            misp_object.uuid = uuid
+            self.misp_event.add_object(**misp_object)
+
+    ################################################################################
+    #                              Utility functions                               #
+    ################################################################################
+
+    def __create_attribute_check_category(self, line, indexes):
+        attribute = self.__create_standard_attribute(line, indexes)
+        self.__check_category(attribute)
+        return attribute
+
+    def __create_attribute_check_category_and_ids(self, line, indexes):
+        attribute = self.__create_attribute_with_ids(line, indexes)
+        self.__check_category(attribute)
+        return attribute
+
+    def __create_attribute_check_category_and_tags(self, line, indexes):
+        attribute = self.__create_attribute_with_tags(line, indexes)
+        self.__check_category(attribute)
+        return attribute
+
+    def __create_attribute_check_category_with_ids_and_tags(self, line, indexes):
+        attribute = self.__create_attribute_with_ids_and_tags(line, indexes)
+        self.__check_category(attribute)
+        return attribute
+
+    def __create_attribute_with_ids(self, line, indexes):
+        attribute = self.__create_standard_attribute(line, indexes)
+        self.__deal_with_ids(attribute)
+        return attribute
+
+    def __create_attribute_with_ids_and_tags(self, line, indexes):
+        attribute = self.__create_standard_attribute(line, indexes)
+        self.__deal_with_ids(attribute)
+        self.__deal_with_tags(attribute)
+        return attribute
+
+    def __create_attribute_with_tags(self, line, indexes):
+        attribute = self.__create_standard_attribute(line, indexes)
+        self.__deal_with_tags(attribute)
+        return attribute
+
+    def __create_standard_attribute(self, line, indexes):
+        return {self.header[index]: line[index] for index in indexes if line[index]}
+
+    def __check_category(self, attribute):
+        category = attribute['category']
+        if category in self.categories:
+            return
+        if category.capitalize() in self.categories:
+            attribute['category'] = category.capitalize()
+            return
+        del attribute['category']
+
+    @staticmethod
+    def __deal_with_ids(attribute):
+        attribute['to_ids'] = True if attribute['to_ids'] == '1' else False
+
+    @staticmethod
+    def __deal_with_tags(attribute):
+        if 'Tag' in attribute.keys():
+            attribute['Tag'] = [{'name': tag.strip()} for tag in attribute['Tag'].split(',')]
+
+    def __get_score(self):
+        score = 1 if 'to_ids' in self.header else 0
+        if 'attribute_tag' in self.header:
+            score += 2
+        if 'category' in self.header:
+            score += 4
+        return score
+
+    def __finalize_results(self):
+        event = self.misp_event.to_dict()
+        self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+
+
+def __any_mandatory_misp_field(header):
+    return any(field in header for field in ('type', 'value'))
+
+
+def __special_parsing(data, delimiter):
+    return list(tuple(part.strip() for part in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
+
+
+def __standard_parsing(data):
+    return list(tuple(part.strip() for part in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
+
+
+def dict_handler(request: dict):
+    if request.get('data'):
+        try:
+            data = base64.b64decode(request['data']).decode('utf-8')
+        except UnicodeDecodeError:
+            misperrors['error'] = "Input is not valid UTF-8"
+            return misperrors
+    else:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+    has_header = request['config'].get('has_header')
+    has_header = True if has_header == '1' else False
+    header = request['config']['header'].split(',') if request['config'].get('header').strip() else []
+    delimiter = request['config']['special_delimiter'] if request['config'].get('special_delimiter').strip() else ','
+    data = __standard_parsing(data) if delimiter == ',' else __special_parsing(data, delimiter)
+    if not header:
+        if has_header:
+            header = data.pop(0)
+        else:
+            misperrors['error'] = "Configuration error. Provide a header or use the one within the csv file and tick the checkbox 'Has_header'."
+            return misperrors
+    else:
+        header = [h.strip() for h in header]
+        if has_header:
+            del data[0]
+    if header == misp_standard_csv_header or header == misp_extended_csv_header:
+        header = misp_standard_csv_header
+
+    description = MISPEvent().describe_types
+    misp_types = description['types']
+    for h in header:
+        if not any((h in misp_types, h in misp_extended_csv_header, h in ('', ' ', '_', 'object_id'))):
+            misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h)
+            return misperrors
+    from_misp = all((h in misp_extended_csv_header or h in ('', ' ', '_', 'object_id') for h in header))
+    if from_misp:
+        if not __any_mandatory_misp_field(header):
+            misperrors['error'] = 'Please make sure the data you try to import can be identified with a type/value combinaison.'
+            return misperrors
+    else:
+        if __any_mandatory_misp_field(header):
+            wrong_types = tuple(wrong_type for wrong_type in ('type', 'value') if wrong_type in header)
+            misperrors['error'] = 'Error with the following header: {}. It contains the following field(s): {}, which is(are) already provided by the usage of at least on MISP attribute type in the header.'.format(header, 'and'.join(wrong_types))
+            return misperrors
+    csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, misp_types, description['categories'])
+    # build the attributes
+    result = csv_parser.parse_csv()
+    if 'error' in result:
+        return result
+    return {'results': csv_parser.results}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo