misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/export_mod/threat_connect_export.py

@@ -0,0 +1,125 @@
+"""
+Export module for converting MISP events into ThreatConnect Structured Import files. This export data is meant to be used with the "Structured Import" ability of ThreatConnect.
+
+Source: http://kb.threatconnect.com/customer/en/portal/articles/1912599-using-structured-import/
+Source: http://kb.threatconnect.com/customer/en/portal/articles/2092925-the-threatconnect-data-model/
+"""
+import base64
+import csv
+import io
+import json
+import logging
+
+misperrors = {"error": "Error"}
+
+moduleinfo = {
+    'version': '0.1',
+    'author': 'CenturyLink CIRT',
+    'description': 'Module to export a structured CSV file for uploading to ThreatConnect.',
+    'module-type': ['export'],
+    'name': 'ThreadConnect Export',
+    'logo': 'threatconnect.png',
+    'requirements': ['csv'],
+    'features': 'The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module configuration, the source of data they export, because it is required by the output format.',
+    'references': ['https://www.threatconnect.com'],
+    'input': 'MISP Event attributes',
+    'output': 'ThreatConnect CSV format file',
+}
+
+# config fields expected from the MISP administrator
+#   Default_Source: The source of the data. Typically this won't be changed from the default
+moduleconfig = ["Default_Source"]
+
+# Map of MISP fields => ThreatConnect fields
+fieldmap = {
+    "domain": "Host",
+    "domain|ip": "Host|Address",
+    "hostname": "Host",
+    "ip-src": "Address",
+    "ip-dst": "Address",
+    "ip-src|port": "Address",
+    "ip-dst|port": "Address",
+    "whois-registrant-email": "EmailAddress",
+    "email-src": "EmailAddress",
+    "email-dst": "EmailAddress",
+    "url": "URL",
+    "md5": "File",
+    "filename|md5": "File"
+}
+
+# combine all the MISP fields from fieldmap into one big list
+mispattributes = {
+    "input": list(fieldmap.keys())
+}
+
+
+def handler(q=False):
+    """
+    Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format.
+    Input
+        q: Query dictionary
+    """
+    if q is False or not q:
+        return False
+
+    # Check if we were given a configuration
+    request = json.loads(q)
+    config = request.get("config", {"Default_Source": ""})
+    logging.info("Setting config to: %s", config)
+
+    response = io.StringIO()
+    writer = csv.DictWriter(response, fieldnames=["Type", "Value", "Source", "Description"])
+    writer.writeheader()
+
+    # start parsing MISP data
+    for event in request["data"]:
+        for attribute in event["Attribute"]:
+            if attribute["type"] in mispattributes["input"]:
+                logging.debug("Adding %s to structured CSV export of ThreatConnectExport", attribute["value"])
+                if "|" in attribute["type"]:
+                    # if the attribute type has multiple values, line it up with the corresponding ThreatConnect values in fieldmap
+                    indicators = tuple(attribute["value"].split("|"))
+                    tc_types = tuple(fieldmap[attribute["type"]].split("|"))
+                    for i, indicator in enumerate(indicators):
+                        writer.writerow({
+                            "Type": tc_types[i],
+                            "Value": indicator,
+                            "Source": config["Default_Source"],
+                            "Description": attribute["comment"]
+                        })
+                else:
+                    writer.writerow({
+                        "Type": fieldmap[attribute["type"]],
+                        "Value": attribute["value"],
+                        "Source": config["Default_Source"],
+                        "Description": attribute["comment"]
+                    })
+
+    return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')}
+
+
+def introspection():
+    """
+    Relay the supported attributes to MISP.
+    No Input
+    Output
+        Dictionary of supported MISP attributes
+    """
+    modulesetup = {
+        "responseType": "application/txt",
+        "outputFileExtension": "csv",
+        "userConfig": {},
+        "inputSource": []
+    }
+    return modulesetup
+
+
+def version():
+    """
+    Relay module version and associated metadata to MISP.
+    No Input
+    Output
+        moduleinfo: metadata output containing all potential configuration values
+    """
+    moduleinfo["config"] = moduleconfig
+    return moduleinfo

misp_modules/modules/export_mod/virustotal_collections.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+# Copyright 2022 Google Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Creates a VT Collection with indicators present in a given event."""
+
+import base64
+import json
+import requests
+
+misperrors = {
+  'error': 'Error'
+}
+
+mispattributes = {
+    'input': [
+        'hostname',
+        'domain',
+        'ip-src',
+        'ip-dst',
+        'md5',
+        'sha1',
+        'sha256',
+        'url'
+    ],
+    'format': 'misp_standard',
+    'responseType': 'application/txt',
+    'outputFileExtension': 'txt',
+}
+
+moduleinfo = {
+    'version': '1.0',
+    'author': 'VirusTotal',
+    'description': 'Creates a VT Collection from an event iocs.',
+    'module-type': ['export'],
+    'name': 'VirusTotal Collections Export',
+    'logo': 'virustotal.png',
+    'requirements': ['An access to the VirusTotal API (apikey).'],
+    'features': 'This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button.',
+    'references': ['https://www.virustotal.com/', 'https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html'],
+    'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.',
+    'output': 'A VirusTotal collection in VT.',
+}
+
+moduleconfig = [
+    'vt_api_key',
+    'proxy_host',
+    'proxy_port',
+    'proxy_username',
+    'proxy_password'
+]
+
+
+class VTError(Exception):
+  "Exception class to map vt api response errors."
+  pass
+
+
+def create_collection(api_key, event_data):
+  headers = {
+      'x-apikey': api_key,
+      'content-type': 'application/json',
+      'x-tool': 'MISPModuleVirusTotalCollectionExport',
+  }
+
+  response = requests.post('https://www.virustotal.com/api/v3/integrations/misp/collections',
+                           headers=headers,
+                           json=event_data)
+
+  uuid = event_data['Event']['uuid']
+  response_data = response.json()
+
+  if response.status_code == 200:
+    col_id = response_data['data']['id']
+    return f'{uuid}: https://www.virustotal.com/gui/collection/{col_id}/iocs'
+
+  error = response_data['error']['message']
+  if response.status_code == 400:
+    return f'{uuid}: {error}'
+  else:
+    misperrors['error'] = error
+    raise VTError(error)
+
+
+def normalize_misp_data(data):
+  normalized_data = {'Event': data.pop('Event', {})}
+  for attr_key in data:
+    if isinstance(data[attr_key], list) or isinstance(data[attr_key], dict):
+      if attr_key == 'EventTag':
+        normalized_data['Event']['Tag'] = [tag['Tag'] for tag in data[attr_key]]
+      else:
+        normalized_data['Event'][attr_key] = data[attr_key]
+
+  return normalized_data
+
+
+def handler(q=False):
+  request = json.loads(q)
+
+  if not request.get('config') or not request['config'].get('vt_api_key'):
+    misperrors['error'] = 'A VirusTotal api key is required for this module.'
+    return misperrors
+
+  config = request['config']
+  data = request['data']
+  responses = []
+
+  try:
+    for event_data in data:
+      normalized_event = normalize_misp_data(event_data)
+      responses.append(create_collection(config.get('vt_api_key'),
+                                         normalized_event))
+
+    output = '\n'.join(responses)
+    return {
+        "response": [],
+        "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8'),
+    }
+  except VTError:
+    return misperrors
+
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/export_mod/vt_graph.py

@@ -0,0 +1,120 @@
+'''Export MISP event to VirusTotal Graph.'''
+
+
+import base64
+import json
+from vt_graph_parser.importers.pymisp_response import from_pymisp_response
+
+
+misperrors = {
+    'error': 'Error'
+}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'VirusTotal',
+    'description': 'This module is used to create a VirusTotal Graph from a MISP event.',
+    'module-type': ['export'],
+    'name': 'VirusTotal Graph Export',
+    'logo': 'virustotal.png',
+    'requirements': ['vt_graph_api, the python library to query the VirusTotal graph API'],
+    'features': 'The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.',
+    'references': ['https://www.virustotal.com/gui/graph-overview'],
+    'input': 'A MISP event.',
+    'output': 'Link of the VirusTotal Graph created for the event.',
+}
+mispattributes = {
+    'input': [
+        'hostname',
+        'domain',
+        'ip-src',
+        'ip-dst',
+        'md5',
+        'sha1',
+        'sha256',
+        'url',
+        'filename|md5',
+        'filename'
+    ]
+}
+moduleconfig = [
+    'vt_api_key',
+    'fetch_information',
+    'private',
+    'fetch_vt_enterprise',
+    'expand_one_level',
+    'user_editors',
+    'user_viewers',
+    'group_editors',
+    'group_viewers'
+]
+
+
+def handler(q=False):
+    """Expansion handler.
+
+    Args:
+      q (bool, optional): module data. Defaults to False.
+
+    Returns:
+      [str]: VirusTotal graph links
+    """
+    if not q:
+        return False
+    request = json.loads(q)
+
+    if not request.get('config') or not request['config'].get('vt_api_key'):
+        misperrors['error'] = 'A VirusTotal api key is required for this module.'
+        return misperrors
+
+    config = request['config']
+
+    api_key = config.get('vt_api_key')
+    fetch_information = config.get('fetch_information') or False
+    private = config.get('private') or False
+    fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False
+    expand_one_level = config.get('expand_one_level') or False
+
+    user_editors = config.get('user_editors')
+    if user_editors:
+        user_editors = user_editors.split(',')
+    user_viewers = config.get('user_viewers')
+    if user_viewers:
+        user_viewers = user_viewers.split(',')
+    group_editors = config.get('group_editors')
+    if group_editors:
+        group_editors = group_editors.split(',')
+    group_viewers = config.get('group_viewers')
+    if group_viewers:
+        group_viewers = group_viewers.split(',')
+
+    graphs = from_pymisp_response(
+        request, api_key, fetch_information=fetch_information,
+        private=private, fetch_vt_enterprise=fetch_vt_enterprise,
+        user_editors=user_editors, user_viewers=user_viewers,
+        group_editors=group_editors, group_viewers=group_viewers,
+        expand_node_one_level=expand_one_level)
+    links = []
+
+    for graph in graphs:
+        graph.save_graph()
+        links.append(graph.get_ui_link())
+
+    # This file will contains one VirusTotal graph link for each exported event
+    file_data = str(base64.b64encode(
+        bytes('\n'.join(links), 'utf-8')), 'utf-8')
+    return {'response': [], 'data': file_data}
+
+
+def introspection():
+    modulesetup = {
+        'responseType': 'application/txt',
+        'outputFileExtension': 'txt',
+        'userConfig': {},
+        'inputSource': []
+    }
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/export_mod/yara_export.py

@@ -0,0 +1,290 @@
+import json
+import base64
+import re
+try:
+    import yara
+except (OSError, ImportError):
+    print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.")
+
+
+misperrors = {'error': 'Error'}
+
+
+userConfig = {
+
+}
+
+moduleconfig = []
+
+# fixed for now, options in the future:
+# event, attribute, event-collection, attribute-collection
+inputSource = ['event']
+
+outputFileExtension = 'yara'
+responseType = 'text/plain'
+
+
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Christophe Vandeplas',
+    'description': 'This module is used to export MISP events to YARA.',
+    'module-type': ['export'],
+    'name': 'YARA Rule Export',
+    'logo': 'yara.png',
+    'requirements': ['yara-python python library'],
+    'features': 'The module will dynamically generate YARA rules for attributes that are marked as to IDS. Basic metadata about the event is added to the rule.\nAttributes that are already YARA rules are also exported, with a rewritten rule name.',
+    'references': ['https://virustotal.github.io/yara/'],
+    'input': 'Attributes and Objects.',
+    'output': 'A YARA file that can be used with the YARA scanning tool.',
+}
+
+
+class YaraRule():
+    def __init__(self, name):
+        self.name = name
+        self.strings = {}
+        self.conditions = []
+        self.meta = {}
+
+    def add_string(self, type_: str, s: str):
+        type_clean = ''.join(c if c.isalnum() or c == '_' else '_' for c in type_)
+        if type_clean not in self.strings:
+            self.strings[type_clean] = []
+        self.strings[type_clean].append(s)
+
+    def add_condition(self, condition: str):
+        self.conditions.append(condition)
+
+    def add_meta(self, key: str, value: str):
+        if key not in self.meta:
+            self.meta[key] = []
+        self.meta[key].append(value)
+
+    def __str__(self):
+        if len(self.strings) == 0 and len(self.conditions) == 0:
+            return "\n"  # no strings, so no rule
+
+        result = []
+        result.append(f"rule {self.name} {{")
+
+        result.append("    meta:")
+        for key, values in self.meta.items():
+            i = 0
+            if len(values) == 1:
+                result.append(f"        {key} = \"{values[0]}\"")
+                continue
+            for value in values:
+                result.append(f"        {key}_{i} = \"{value}\"")
+                i += 1
+
+        result.append("    strings:")
+        for key, values in self.strings.items():
+            i = 0
+            for value in values:
+                result.append(f"        ${key}_{i} = \"{value}\"")
+                i += 1
+
+        result.append("    condition:")
+        if len(self.conditions) == 0:
+            result.append("        any of them")
+        for condition in self.conditions:
+            result.append(f"        {condition}")
+
+        result.append("}")
+        result.append("")
+        return '\n'.join(result)
+
+
+def handle_string(yara_rules: list, yr: YaraRule, attribute: dict):
+    if not attribute['to_ids']:  # skip non IDS attributes
+        return
+    yr.add_string(attribute['type'], attribute['value'])
+    return
+
+
+def handle_combined(yara_rules: list, yr: YaraRule, attribute: dict):
+    if not attribute['to_ids']:  # skip non IDS attributes
+        return
+    type_1, type_2 = attribute['type'].split('|')
+    value_1, value_2 = attribute['value'].split('|')
+    try:
+        handlers[type_1](yara_rules, yr, type_1, value_1)
+    except KeyError:
+        # ignore unsupported types
+        pass
+    try:
+        handlers[type_2](yara_rules, yr, type_2, value_2)
+    except KeyError:
+        # ignore unsupported types
+        pass
+
+
+def handle_yara(yara_rules: list, yr: YaraRule, attribute: dict):
+    # do not check for to_ids, as we want to always export the Yara rule
+    # split out as a separate rule, and rewrite the rule name
+    value = re.sub('^[ \t]*rule ', 'rule MISP_e{}_'.format(attribute['event_id']), attribute['value'], flags=re.MULTILINE)
+    # cleanup dirty stuff from people
+    substitutions = (('”', '"'),
+                     ('“', '"'),
+                     ('″', '"'),
+                     ('`', "'"),
+                     ('\r', ''),
+                     ('Rule ', 'rule ')  # some people write this with the wrong case
+                     # ('$ ', '$'),    # this breaks rules
+                     # ('\t\t', '\n'), # this breaks rules
+                     )
+    for substitution in substitutions:
+        if substitution[0] in value:
+            value = value.replace(substitution[0], substitution[1])
+
+    # we may ignore any global rules as they might disable everything
+    # on the other hand we're only processing one event...
+    # if 'global rule' in value:
+    #     return
+
+    # private rules need some more rewriting
+    if 'private rule' in value:
+        priv_rules = re.findall(r'private rule (\w+)', value, flags=re.MULTILINE)
+        for priv_rule in priv_rules:
+            value = re.sub(priv_rule, 'MISP_e{}_{}'.format(attribute['event_id'], priv_rule), value, flags=re.MULTILINE)
+
+    # compile the yara rule to confirm it's validity
+    try:
+        yara.compile(source=value)
+    except Exception:
+        # skip rules that do not compile
+        return
+
+    # all checks done, add the rule
+    yara_rules.append(value)
+    return
+
+
+def handle_malware_sample(yara_rules: list, yr: YaraRule, attribute: dict):
+    if not attribute['to_ids']:  # skip non IDS attributes
+        return
+    handle_combined(yara_rules, yr, 'filename|md5', attribute['value'])
+
+
+def handle_meta(yara_rules: list, yr: YaraRule, attribute: dict):
+    yr.add_meta(attribute['type'], attribute['value'])
+    return
+
+
+handlers = {
+    'yara': handle_yara,
+    'hostname': handle_string,
+    'hostname|port': handle_combined,
+    'domain': handle_string,
+    'domain|ip': handle_combined,
+    'ip': handle_string,
+    'ip-src': handle_string,
+    'ip-dst': handle_string,
+    'ip-dst|port': handle_combined,  # we could also handle_string, which would be more specific. Less false positives, but less true positives too...
+    'ip-src|port': handle_combined,
+    'url': handle_string,
+    'email': handle_string,
+    'email-src': handle_string,
+    'email-dst': handle_string,
+    'email-subject': handle_string,
+    'email-attachment': handle_string,
+    'email-header': handle_string,
+    'email-reply-to': handle_string,
+    'email-x-mailer': handle_string,
+    'email-mime-boundary': handle_string,
+    'email-thread-index': handle_string,
+    'email-message-id': handle_string,
+    'filename': handle_string,
+    'filename|md5': handle_combined,
+    'filename|sha1': handle_combined,
+    'filename|sha256': handle_combined,
+    'filename|authentihash': handle_combined,
+    'filename|vhash': handle_combined,
+    'filename|ssdeep': handle_combined,
+    'filename|imphash': handle_combined,
+    'filename|impfuzzy': handle_combined,
+    'filename|pehash': handle_combined,
+    'filename|sha224': handle_combined,
+    'filename|sha384': handle_combined,
+    'filename|sha512': handle_combined,
+    'filename|sha512/224': handle_combined,
+    'filename|sha512/256': handle_combined,
+    'filename|sha3-224': handle_combined,
+    'filename|sha3-256': handle_combined,
+    'filename|sha3-384': handle_combined,
+    'filename|sha3-512': handle_combined,
+    'filename|tlsh': handle_combined,
+    'malware-sample': handle_malware_sample,
+    'pattern-in-file': handle_string,
+    'pattern-in-traffic': handle_string,
+    'pattern-in-memory': handle_string,
+    'link': handle_meta
+}
+
+# auto-generate the list of types to use
+types_to_use = handlers.keys()
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    yara_rules = []
+    for event in request["data"]:
+        event_info_clean = ''.join(c if c.isalnum() or c == '_' else '_' for c in event['Event']['info'])
+        yr = YaraRule(f"MISP_e{event['Event']['id']}_{event_info_clean}")
+
+        yr.add_meta('description', event['Event']['info'])
+        yr.add_meta('author', f"MISP - {event['Orgc']['name']}")
+        yr.add_meta('misp_event_date', event['Event']['date'])
+        yr.add_meta('misp_event_id', event['Event']['id'])
+        yr.add_meta('misp_event_uuid', event['Event']['uuid'])
+
+        for attribute in event.get("Attribute", []):
+            try:
+                handlers[attribute['type']](yara_rules, yr, attribute)
+            except KeyError:
+                # ignore unsupported types
+                pass
+        for obj in event.get("Object", []):
+            for attribute in obj["Attribute"]:
+                try:
+                    handlers[attribute['type']](yara_rules, yr, attribute)
+                except KeyError:
+                    # ignore unsupported types
+                    pass
+        yara_rules.append(str(yr))
+    r = {"response": [], "data": str(base64.b64encode(bytes('\n'.join(yara_rules), 'utf-8')), 'utf-8')}
+
+    return r
+
+
+def introspection():
+    modulesetup = {}
+    try:
+        responseType
+        modulesetup['responseType'] = responseType
+    except NameError:
+        pass
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    try:
+        outputFileExtension
+        modulesetup['outputFileExtension'] = outputFileExtension
+    except NameError:
+        pass
+    try:
+        inputSource
+        modulesetup['inputSource'] = inputSource
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/import_mod/__init__.py

@@ -0,0 +1,22 @@
+import os
+import sys
+sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
+
+__all__ = [
+    'vmray_import',
+    'lastline_import',
+    'ocr',
+    'cuckooimport',
+    'goamlimport',
+    'email_import',
+    'mispjson',
+    'openiocimport',
+    'threatanalyzer_import',
+    'csvimport',
+    'cof2misp',
+    'joe_import',
+    'taxii21',
+    'url_import',
+    'vmray_summary_json_import',
+    'import_blueprint'
+]