misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/expansion/onyphe_full.py

@@ -0,0 +1,387 @@
+# -*- coding: utf-8 -*-
+
+import json
+try:
+    from onyphe import Onyphe
+except ImportError:
+    print("pyonyphe module not installed.")
+
+misperrors = {'error': 'Error'}
+
+mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
+                  'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {
+    'version': '1',
+    'author': 'Sebastien Larinier @sebdraven',
+    'description': 'Module to process a full query on Onyphe.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'Onyphe Full Lookup',
+    'logo': 'onyphe.jpg',
+    'requirements': ['onyphe python library', 'An access to the Onyphe API (apikey)'],
+    'features': 'This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted.\n\nThe parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed.',
+    'references': ['https://www.onyphe.io/', 'https://github.com/sebdraven/pyonyphe'],
+    'input': 'A domain, hostname or IP address MISP attribute.',
+    'output': 'MISP attributes fetched from the Onyphe query.',
+}
+
+# config fields that your code expects from the site admin
+moduleconfig = ['apikey']
+
+
+def handler(q=False):
+    if q:
+
+        request = json.loads(q)
+
+        if not request.get('config') or not request['config'].get('apikey'):
+            misperrors['error'] = 'Onyphe authentication is missing'
+            return misperrors
+
+        api = Onyphe(request['config'].get('apikey'))
+
+        if not api:
+            misperrors['error'] = 'Onyphe Error instance api'
+
+        ip = ''
+        if request.get('ip-src'):
+            ip = request['ip-src']
+            return handle_ip(api, ip, misperrors)
+        elif request.get('ip-dst'):
+            ip = request['ip-dst']
+            return handle_ip(api, ip, misperrors)
+        elif request.get('domain'):
+            domain = request['domain']
+            return handle_domain(api, domain, misperrors)
+        elif request.get('hostname'):
+            hostname = request['hostname']
+            return handle_domain(api, hostname, misperrors)
+        else:
+            misperrors['error'] = "Unsupported attributes type"
+            return misperrors
+    else:
+        return False
+
+
+def handle_domain(api, domain, misperrors):
+    result_filtered = {"results": []}
+
+    r, status_ok = expand_pastries(api, misperrors, domain=domain)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error pastries result'
+        return misperrors
+
+    r, status_ok = expand_datascan(api, misperrors, domain=domain)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error datascan result '
+        return misperrors
+
+    r, status_ok = expand_threatlist(api, misperrors, domain=domain)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error threat list'
+        return misperrors
+
+    return result_filtered
+
+
+def handle_ip(api, ip, misperrors):
+    result_filtered = {"results": []}
+
+    r, status_ok = expand_syscan(api, ip, misperrors)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = "Error syscan result"
+
+    r, status_ok = expand_pastries(api, misperrors, ip=ip)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error pastries result'
+        return misperrors
+
+    r, status_ok = expand_datascan(api, misperrors, ip=ip)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error datascan result '
+        return misperrors
+
+    r, status_ok = expand_forward(api, ip, misperrors)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error forward result'
+        return misperrors
+
+    r, status_ok = expand_reverse(api, ip, misperrors)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error reverse result'
+        return misperrors
+
+    r, status_ok = expand_threatlist(api, misperrors, ip=ip)
+
+    if status_ok:
+        result_filtered['results'].extend(r)
+    else:
+        misperrors['error'] = 'Error threat list'
+        return misperrors
+
+    return result_filtered
+
+
+def expand_syscan(api, ip, misperror):
+    status_ok = False
+    r = []
+    asn_list = []
+    os_list = []
+    geoloc = []
+    orgs = []
+    results = api.synscan(ip)
+
+    if results['status'] == 'ok':
+        status_ok = True
+        for elem in results['results']:
+            asn_list.append(elem['asn'])
+            os_target = elem['os']
+            geoloc.append(elem['location'])
+            orgs.append(elem['organization'])
+            if os_target != 'Unknown' and os_target != 'Undefined':
+                os_list.append(elem['os'])
+
+        r.append({'types': ['target-machine'],
+                  'values': list(set(os_list)),
+                  'categories': ['Targeting data'],
+                  'comment': 'OS found on %s with synscan of Onyphe' % ip})
+
+        r.append({'types': ['target-location'],
+                  'values': list(set(geoloc)),
+                  'categories': ['Targeting data'],
+                  'comment': 'Geolocalisation of %s found with synscan of Onyphe'
+                  % ip
+                  })
+
+        r.append({'types': ['target-org'],
+                  'values': list(set(orgs)),
+                  'categories': ['Targeting data'],
+                  'comment': 'Organisations of %s found with synscan of Onyphe'
+                  % ip
+                  })
+
+        r.append({'types': ['AS'],
+                  'values': list(set(asn_list)),
+                  'categories': ['Network activity'],
+                  'comment': 'As number of %s found with synscan of Onyphe' % ip
+                  })
+
+    return r, status_ok
+
+
+def expand_datascan(api, misperror, **kwargs):
+    status_ok = False
+    r = []
+    # ip = ''
+    query = ''
+    asn_list = []
+    geoloc = []
+    orgs = []
+    ports = []
+
+    if 'ip' in kwargs:
+        query = kwargs.get('ip')
+        results = api.datascan(query)
+    else:
+        query = kwargs.get('domain')
+        results = api.search_datascan('domain:%s' % query)
+
+    if results['status'] == 'ok':
+        status_ok = True
+        for elem in results['results']:
+            asn_list.append(elem['asn'])
+            geoloc.append(elem['location'])
+            orgs.append(elem['organization'])
+            ports.append(elem['port'])
+
+        r.append({'types': ['port'],
+                  'values': list(set(ports)),
+                  'categories': ['Other'],
+                  'comment': 'Ports of %s found with datascan of Onyphe'
+                             % query
+                  })
+
+        r.append({'types': ['target-location'],
+                  'values': list(set(geoloc)),
+                  'categories': ['Targeting data'],
+                  'comment': 'Geolocalisation of %s found with synscan of Onyphe'
+                             % query
+                  })
+
+        r.append({'types': ['target-org'],
+                  'values': list(set(orgs)),
+                  'categories': ['Targeting data'],
+                  'comment': 'Organisations of %s found with synscan of Onyphe'
+                             % query
+                  })
+
+        r.append({'types': ['AS'],
+                  'values': list(set(asn_list)),
+                  'categories': ['Network activity'],
+                  'comment': 'As number of %s found with synscan of Onyphe' % query
+                  })
+    return r, status_ok
+
+
+def expand_reverse(api, ip, misperror):
+    status_ok = False
+    r = None
+    status_ok = False
+    r = []
+    results = api.reverse(ip)
+
+    domains_reverse = []
+
+    domains = []
+    if results['status'] == 'ok':
+        status_ok = True
+
+    for elem in results['results']:
+        domains_reverse.append(elem['reverse'])
+        domains.append(elem['domain'])
+
+    r.append({'types': ['domain'],
+              'values': list(set(domains)),
+              'categories': ['Network activity'],
+              'comment': 'Domains of %s from forward service of Onyphe' % ip})
+
+    r.append({'types': ['domain'],
+              'values': list(set(domains_reverse)),
+              'categories': ['Network activity'],
+              'comment': 'Reverse Domains of %s from forward service of Onyphe' % ip})
+    return r, status_ok
+
+
+def expand_forward(api, ip, misperror):
+    status_ok = False
+    r = []
+    results = api.forward(ip)
+
+    domains_forward = []
+
+    domains = []
+    if results['status'] == 'ok':
+        status_ok = True
+
+    for elem in results['results']:
+        domains_forward.append(elem['forward'])
+        domains.append(elem['domain'])
+
+    r.append({'types': ['domain'],
+              'values': list(set(domains)),
+              'categories': ['Network activity'],
+              'comment': 'Domains of %s from forward service of Onyphe' % ip})
+
+    r.append({'types': ['domain'],
+              'values': list(set(domains_forward)),
+              'categories': ['Network activity'],
+              'comment': 'Forward Domains of %s from forward service of Onyphe' % ip})
+    return r, status_ok
+
+
+def expand_pastries(api, misperror, **kwargs):
+    status_ok = False
+    r = []
+
+    query = None
+    result = None
+    urls_pasties = []
+    domains = []
+    ips = []
+    if 'ip' in kwargs:
+        query = kwargs.get('ip')
+        result = api.pastries(query)
+    if 'domain' in kwargs:
+        query = kwargs.get('domain')
+        result = api.search_pastries('domain:%s' % query)
+
+    if result['status'] == 'ok':
+        status_ok = True
+        for item in result['results']:
+            if item['@category'] == 'pastries':
+                if item['source'] == 'pastebin':
+                    urls_pasties.append('https://pastebin.com/raw/%s' % item['key'])
+
+                    if 'domain' in item:
+                        domains.extend(item['domain'])
+                    if 'ip' in item:
+                        ips.extend(item['ip'])
+                    if 'hostname' in item:
+                        domains.extend(item['hostname'])
+
+        r.append({'types': ['url'],
+                  'values': urls_pasties,
+                  'categories': ['External analysis'],
+                  'comment': 'URLs of pasties where %s has found' % query})
+        r.append({'types': ['domain'], 'values': list(set(domains)),
+                  'categories': ['Network activity'],
+                  'comment': 'Domains found in pasties of Onyphe'})
+
+        r.append({'types': ['ip-dst'], 'values': list(set(ips)),
+                  'categories': ['Network activity'],
+                  'comment': 'IPs found in pasties of Onyphe'})
+
+    return r, status_ok
+
+
+def expand_threatlist(api, misperror, **kwargs):
+    status_ok = False
+    r = []
+
+    query = None
+
+    threat_list = []
+
+    if 'ip' in kwargs:
+        query = kwargs.get('ip')
+        results = api.threatlist(query)
+    else:
+        query = kwargs.get('domain')
+        results = api.search_threatlist('domain:%s' % query)
+
+    if results['status'] == 'ok':
+        status_ok = True
+        threat_list = ['seen %s on %s ' % (item['seen_date'], item['threatlist'])
+                       for item in results['results']]
+
+        r.append({'types': ['comment'],
+                  'categories': ['Other'],
+                  'values': threat_list,
+                  'comment': '%s is present in threatlist' % query
+                  })
+
+    return r, status_ok
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/otx.py

@@ -0,0 +1,167 @@
+import json
+import requests
+import re
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ["hostname", "domain", "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"],
+                  'output': ["domain", "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "email"]
+                  }
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {
+    'version': '1',
+    'author': 'chrisdoman',
+    'description': 'Module to get information from AlienVault OTX.',
+    'module-type': ['expansion'],
+    'name': 'AlienVault OTX Lookup',
+    'logo': 'otx.png',
+    'requirements': ['An access to the OTX API (apikey)'],
+    'features': 'This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.',
+    'references': ['https://www.alienvault.com/open-threat-exchange'],
+    'input': 'A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512',
+    'output': 'MISP attributes mapped from the result of the query on OTX, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- email',
+}
+
+# We're not actually using the API key yet
+moduleconfig = ["apikey"]
+
+
+# Avoid adding windows update to enrichment etc.
+def isBlacklisted(value):
+    blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com']
+
+    for b in blacklist:
+        if value in b:
+            return False
+
+    return True
+
+
+def valid_ip(ip):
+    m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip)
+    return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups()))
+
+
+def findAll(data, keys):
+    a = []
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if key == keys:
+                a.append(value)
+            else:
+                if isinstance(value, (dict, list)):
+                    a.extend(findAll(value, keys))
+    if isinstance(data, list):
+        for i in data:
+            a.extend(findAll(i, keys))
+    return a
+
+
+def valid_email(email):
+    return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email))
+
+
+def handler(q=False):
+    if q is False:
+        return False
+
+    q = json.loads(q)
+
+    key = q["config"]["apikey"]
+
+    r = {"results": []}
+
+    if "ip-src" in q:
+        r["results"] += getIP(q["ip-src"], key)
+    if "ip-dst" in q:
+        r["results"] += getIP(q["ip-dst"], key)
+    if "domain" in q:
+        r["results"] += getDomain(q["domain"], key)
+    if 'hostname' in q:
+        r["results"] += getDomain(q['hostname'], key)
+    if 'md5' in q:
+        r["results"] += getHash(q['md5'], key)
+    if 'sha1' in q:
+        r["results"] += getHash(q['sha1'], key)
+    if 'sha256' in q:
+        r["results"] += getHash(q['sha256'], key)
+    if 'sha512' in q:
+        r["results"] += getHash(q['sha512'], key)
+
+    uniq = []
+    for res in r["results"]:
+        if res not in uniq:
+            uniq.append(res)
+    r["results"] = uniq
+    return r
+
+
+def getHash(_hash, key):
+
+    ret = []
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text)
+
+    for ip in findAll(req, "dst"):
+        if not isBlacklisted(ip) and valid_ip(ip):
+            ret.append({"types": ["ip-dst", "ip-src"], "values": [ip]})
+
+    for domain in findAll(req, "hostname"):
+        if "." in domain and not isBlacklisted(domain):
+            ret.append({"types": ["hostname"], "values": [domain]})
+
+    return ret
+
+
+def getIP(ip, key):
+    ret = []
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text)
+
+    for _hash in findAll(req, "hash"):
+        ret.append({"types": ["sha256"], "values": [_hash]})
+
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text)
+
+    for hostname in findAll(req, "hostname"):
+        if not isBlacklisted(hostname):
+            ret.append({"types": ["hostname"], "values": [hostname]})
+
+    return ret
+
+
+def getDomain(domain, key):
+
+    ret = []
+
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text)
+
+    for _hash in findAll(req, "hash"):
+        ret.append({"types": ["sha256"], "values": [_hash]})
+
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text)
+
+    for _domain in findAll(req, "domain"):
+        ret.append({"types": ["hostname"], "values": [_domain]})
+
+    for email in findAll(req, "value"):
+        if valid_email(email):
+            ret.append({"types": ["email"], "values": [email]})
+
+    for _domain in findAll(req, "hostname"):
+        if "." in _domain and not isBlacklisted(_domain):
+            ret.append({"types": ["hostname"], "values": [_domain]})
+
+    req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text)
+    for ip in findAll(req, "address"):
+        if valid_ip(ip):
+            ret.append({"types": ["ip-dst"], "values": [ip]})
+
+    return ret
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/passive_ssh.py

@@ -0,0 +1,150 @@
+import json
+import requests
+from . import check_input_attribute, standard_error_message
+from collections import defaultdict
+from pymisp import MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+
+mispattributes = {'input': ['ip-src', 'ip-dst', 'ssh-fingerprint'],
+                  'format': 'misp_standard'}
+
+moduleinfo = {
+    'version': '1',
+    'author': 'Jean-Louis Huynen',
+    'description': 'An expansion module to enrich, SSH key fingerprints and IP addresses with information collected by passive-ssh',
+    'module-type': ['expansion', 'hover'],
+    'name': 'Passive SSH Enrichment',
+    'logo': '',
+    'requirements': [],
+    'features': '',
+    'references': [],
+    'input': '',
+    'output': '',
+}
+
+moduleconfig = ["custom_api_url", "api_user", "api_key"]
+
+passivessh_url = 'https://passivessh.circl.lu/'
+
+host_query = '/host/ssh'
+fingerprint_query = '/fingerprint/all'
+
+
+class PassivesshParser():
+    def __init__(self, attribute, passivesshresult):
+        self.attribute = attribute
+        self.passivesshresult = passivesshresult
+        self.misp_event = MISPEvent()
+        self.misp_event.add_attribute(**attribute)
+        self.references = defaultdict(list)
+
+    def get_result(self):
+        if self.references:
+            self.__build_references()
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in (
+            'Attribute', 'Object') if (key in event and event[key])}
+        return {'results': results}
+
+    def parse_passivessh_information(self):
+        passivessh_object = MISPObject('passive-ssh')
+        if 'first_seen' in self.passivesshresult:
+            passivessh_object.add_attribute(
+                'first_seen', **{'type': 'datetime', 'value': self.passivesshresult['first_seen']})
+        if 'last_seen' in self.passivesshresult:
+            passivessh_object.add_attribute(
+                'last_seen', **{'type': 'datetime', 'value': self.passivesshresult['last_seen']})
+        if 'base64' in self.passivesshresult:
+            passivessh_object.add_attribute(
+                'base64', **{'type': 'text', 'value': self.passivesshresult['base64']})
+        if 'keys' in self.passivesshresult:
+            for key in self.passivesshresult['keys']:
+                passivessh_object.add_attribute(
+                    'fingerprint', **{'type': 'ssh-fingerprint', 'value': key['fingerprint']})
+        if 'hosts' in self.passivesshresult:
+            for host in self.passivesshresult['hosts']:
+                passivessh_object.add_attribute(
+                    'host', **{'type': 'ip-dst', 'value': host})
+
+        passivessh_object.add_reference(self.attribute['uuid'], 'related-to')
+        self.misp_event.add_object(passivessh_object)
+
+    def __build_references(self):
+        for object_uuid, references in self.references.items():
+            for misp_object in self.misp_event.objects:
+                if misp_object.uuid == object_uuid:
+                    for reference in references:
+                        misp_object.add_reference(**reference)
+                    break
+
+
+def check_url(url):
+    return "{}/".format(url) if not url.endswith('/') else url
+
+
+def handler(q=False):
+
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    api_url = check_url(request['config']['custom_api_url']) if request['config'].get(
+        'custom_api_url') else passivessh_url
+
+    if request['config'].get('api_user'):
+        api_user = request['config'].get('api_user')
+    else:
+        misperrors['error'] = 'passive-ssh user required'
+        return misperrors
+    if request['config'].get('api_key'):
+        api_key = request['config'].get('api_key')
+    else:
+        misperrors['error'] = 'passive-ssh password required'
+        return misperrors
+
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute.get('type') == 'ip-src':
+        type = host_query
+        pass
+    elif attribute.get('type') == 'ip-dst':
+        type = host_query
+        pass
+    elif attribute.get('type') == 'ssh-fingerprint':
+        type = fingerprint_query
+        pass
+    else:
+        misperrors['error'] = 'ip is missing.'
+        return misperrors
+
+    r = requests.get("{}{}/{}".format(api_url, type,
+                     attribute['value']), auth=(api_user, api_key))
+
+    if r.status_code == 200:
+        passivesshresult = r.json()
+        if not passivesshresult:
+            misperrors['error'] = 'Empty result'
+            return misperrors
+    elif r.status_code == 404:
+        misperrors['error'] = 'Non existing hash'
+        return misperrors
+    else:
+        misperrors['error'] = 'API not accessible'
+        return misperrors
+
+    parser = PassivesshParser(attribute, passivesshresult)
+    parser.parse_passivessh_information()
+    result = parser.get_result()
+
+    return result
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo