misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/expansion/btc_steroids.py

@@ -0,0 +1,238 @@
+import json
+import requests
+import time
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['btc'], 'output': ['text']}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Sascha Rommelfangen',
+    'description': 'An expansion hover module to get a blockchain balance from a BTC address in MISP.',
+    'module-type': ['hover'],
+    'name': 'BTC Steroids',
+    'logo': 'bitcoin.png',
+    'requirements': [],
+    'features': '',
+    'references': [],
+    'input': 'btc address attribute.',
+    'output': 'Text to describe the blockchain balance and the transactions related to the btc address in input.',
+}
+
+moduleconfig = []
+
+blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/'
+blockchain_balance = 'https://blockchain.info/q/addressbalance/'
+blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/'
+blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}'
+converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}'
+converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit'
+result_text = ""
+g_rate_limit = 300
+start_time = 0
+conversion_rates = {}
+
+
+def get_consumption(output=False):
+    try:
+        req = requests.get(converter_rls)
+        jreq = req.json()
+        minute = str(jreq['Data']['calls_left']['minute'])
+        hour = str(jreq['Data']['calls_left']['hour'])
+    except Exception:
+        minute = str(-1)
+        hour = str(-1)
+    # Debug out for the console
+    print("Calls left this minute / hour: " + minute + " / " + hour)
+    return minute, hour
+
+
+def convert(btc, timestamp):
+    global g_rate_limit
+    global start_time
+    global now
+    global conversion_rates
+    date = time.strftime('%Y-%m-%d', time.localtime(timestamp))
+    # Lookup conversion rates in the cache:
+    if date in conversion_rates:
+        (usd, eur) = conversion_rates[date]
+    else:
+        # If not cached, we have to get the converion rates
+        # We have to be careful with rate limiting on the server side
+        if g_rate_limit == 300:
+            minute, hour = get_consumption()
+        g_rate_limit -= 1
+        now = time.time()
+        # delta = now - start_time
+        # print(g_rate_limit)
+        if g_rate_limit <= 10:
+            minute, hour = get_consumption(output=True)
+            if int(minute) <= 10:
+                # print(minute)
+                # get_consumption(output=True)
+                time.sleep(3)
+            else:
+                mprint(minute)
+                start_time = time.time()
+                g_rate_limit = int(minute)
+        try:
+            req = requests.get(converter.format(timestamp))
+            jreq = req.json()
+            usd = jreq['BTC']['USD']
+            eur = jreq['BTC']['EUR']
+            # Since we have the rates, store them in the cache
+            conversion_rates[date] = (usd, eur)
+        except Exception as ex:
+            mprint(ex)
+            get_consumption(output=True)
+    # Actually convert and return the values
+    u = usd * btc
+    e = eur * btc
+    return u, e
+
+
+def mprint(input):
+    # Prepare the final print
+    global result_text
+    result_text = result_text + "\n" + str(input)
+
+
+def handler(q=False):
+    global result_text
+    global conversion_rates
+    result_text = ""
+    # start_time = time.time()
+    # now = time.time()
+    if q is False:
+        return False
+    request = json.loads(q)
+    click = False
+    # This means the magnifying glass has been clicked
+    if request.get('persistent') == 1:
+        click = True
+    # Otherwise the attribute was only hovered over
+    if request.get('btc'):
+        btc = request['btc']
+    else:
+        return False
+    mprint("\nAddress:\t" + btc)
+    try:
+        req = requests.get(blockchain_all.format(btc, "&limit=50"))
+        jreq = req.json()
+    except Exception:
+        # print(e)
+        print(req.text)
+        result_text = "Not a valid BTC address"
+        r = {
+            'results': [
+                {
+                    'types': ['text'],
+                    'values':[
+                        str(result_text)
+                    ]
+                }
+            ]
+        }
+        return r
+
+    n_tx = jreq['n_tx']
+    balance = float(jreq['final_balance'] / 100000000)
+    rcvd = float(jreq['total_received'] / 100000000)
+    sent = float(jreq['total_sent'] / 100000000)
+    output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)'
+    mprint(output.format(balance, rcvd, sent))
+    if click is False:
+        mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)")
+    else:
+        mprint("Transactions:\t" + str(n_tx))
+    if n_tx > 0:
+        mprint("======================================================================================")
+    i = 0
+    while i < n_tx:
+        if click is False:
+            try:
+                req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
+            except Exception as e:
+                # Lazy retry - cries for a function
+                print(e)
+                time.sleep(3)
+                req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i)))
+            if n_tx > 5:
+                n_tx = 5
+        else:
+            try:
+                req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
+            except Exception as e:
+                # Lazy retry - cries for a function
+                print(e)
+                time.sleep(3)
+                req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i)))
+        jreq = req.json()
+        if jreq['txs']:
+            for transactions in jreq['txs']:
+                sum = 0
+                sum_counter = 0
+                for tx in transactions['inputs']:
+                    script_old = tx['script']
+                    try:
+                        addr_in = tx['prev_out']['addr']
+                    except KeyError:
+                        addr_in = None
+                    try:
+                        prev_out = tx['prev_out']['value']
+                    except KeyError:
+                        prev_out = None
+                    if prev_out != 0 and addr_in == btc:
+                        datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
+                        value = float(tx['prev_out']['value'] / 100000000)
+                        u, e = convert(value, transactions['time'])
+                        mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
+                        if script_old != tx['script']:
+                            i += 1
+                        else:
+                            sum_counter += 1
+                            sum += value
+                if sum_counter > 1:
+                    u, e = convert(sum, transactions['time'])
+                    mprint("\t\t\t\t\t----------------------------------------------")
+                    mprint("#" + str(n_tx - i) + "\t\t\t\t  Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0'))
+                for tx in transactions['out']:
+                    try:
+                        addr_out = tx['addr']
+                    except KeyError:
+                        addr_out = None
+                    try:
+                        prev_out = tx['prev_out']['value']
+                    except KeyError:
+                        prev_out = None
+                    if prev_out != 0 and addr_out == btc:
+                        datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time'])))
+                        value = float(tx['value'] / 100000000)
+                        u, e = convert(value, transactions['time'])
+                        mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
+                        # i += 1
+                i += 1
+
+    r = {
+        'results': [
+            {
+                'types': ['text'],
+                'values':[
+                    str(result_text)
+                ]
+            }
+        ]
+    }
+    # Debug output on the console
+    print(result_text)
+    # Unset the result for the next request
+    result_text = ""
+    return r
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/censys_enrich.py

@@ -0,0 +1,287 @@
+# encoding: utf-8
+import json
+import configparser
+import base64
+import codecs
+import censys.common.config
+from dateutil.parser import isoparse
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+try:
+    #needed in order to overwrite the censys module intent of creating config files in the home folder of the proccess owner
+    #--
+    def get_config_over() -> configparser.ConfigParser:
+        config = configparser.ConfigParser()
+        config[censys.common.config.DEFAULT] = censys.common.config.default_config
+        return config
+    censys.common.config.get_config = get_config_over
+    #--
+
+    from censys.search import CensysHosts
+    from censys.search import CensysCertificates
+    from censys.common.base import *
+except ImportError:
+    print("Censys module not installed. Try 'pip install censys'")
+
+misperrors = {'error': 'Error'}
+moduleconfig = ['api_id', 'api_secret']
+mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port',
+                  'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Loïc Fortemps',
+    'description': 'An expansion module to enrich attributes in MISP by quering the censys.io API',
+    'module-type': ['expansion', 'hover'],
+    'name': 'Censys Enrich',
+    'logo': '',
+    'requirements': ['API credentials to censys.io'],
+    'features': 'This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API.',
+    'references': ['https://www.censys.io'],
+    'input': 'IP, domain or certificate fingerprint (md5, sha1 or sha256)',
+    'output': 'MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details',
+}
+
+api_id = None
+api_secret = None
+
+def handler(q=False):
+    global api_id, api_secret
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    if request.get('config'):
+        if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None):
+            misperrors['error'] = "Censys API credentials are missing"
+            return misperrors
+    else:
+        misperrors['error'] = "Please provide config options"
+        return misperrors
+
+    api_id = request['config']['api_id']
+    api_secret = request['config']['api_secret']
+
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
+        return {'error': 'Unsupported attribute type.'}
+
+    attribute = MISPAttribute()
+    attribute.from_dict(**request['attribute'])
+    # Lists to accomodate multi-types attribute
+    types = list()
+    values = list()
+    results = list()
+
+    if "|" in attribute.type:
+        t_1, t_2 = attribute.type.split('|')
+        v_1, v_2 = attribute.value.split('|')
+        # We cannot use the port information
+        if t_2 == "port":
+            types.append(t_1)
+            values.append(v_1)
+        else:
+            types = [t_1, t_2]
+            values = [v_1, v_2]
+    else:
+        types.append(attribute.type)
+        values.append(attribute.value)
+
+    found = False
+    for t in types:
+        try:
+            value = values.pop(0)
+            # ip, ip-src or ip-dst
+            if t[:2] == "ip":
+                r = CensysHosts(api_id, api_secret).view(value)
+                results.append(parse_response(r, attribute))
+                found = True
+            elif t == 'domain' or t == "hostname":
+                # get ips
+                endpoint = CensysHosts(api_id, api_secret)
+                for r_list in endpoint.search(query=value, per_page=5, pages=1):
+                    for r in r_list:
+                        results.append(parse_response(r, attribute))
+                        found = True
+            elif 'x509-fingerprint-sha256' in t:
+                # use api_v1 as Certificates endpoint in api_v2 doesn't yet provide all the details
+                r = CensysCertificates(api_id, api_secret).view(value)
+                results.append(parse_response(r, attribute))
+                found = True
+        except CensysException as e:
+            misperrors['error'] = "ERROR: param {} / response: {}".format(value, e)
+            return misperrors
+
+    if not found:
+        misperrors['error'] = "Nothing could be found on Censys"
+        return misperrors
+
+    return {'results': remove_duplicates(results)}
+
+
+def parse_response(censys_output, attribute):
+    misp_event = MISPEvent()
+    misp_event.add_attribute(**attribute)
+    # Generic fields (for IP/Websites)
+    if censys_output.get('autonomous_system'):
+        cen_as = censys_output.get('autonomous_system')
+        asn_object = MISPObject('asn')
+        asn_object.add_attribute('asn', value=cen_as.get("asn"))
+        asn_object.add_attribute('description', value=cen_as.get('name'))
+        asn_object.add_attribute('subnet-announced', value=cen_as.get('routed_prefix'))
+        asn_object.add_attribute('country', value=cen_as.get('country_code'))
+        asn_object.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**asn_object)
+
+    if censys_output.get('ip') and len(censys_output.get('services')): #"ports" in censys_output
+        ip_object = MISPObject('ip-port')
+        ip_object.add_attribute('ip', value=censys_output.get('ip'))
+        for serv in censys_output.get('services'):
+            if serv.get('port'):
+                ip_object.add_attribute('dst-port', value=serv.get('port'))
+        ip_object.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**ip_object)
+
+    # We explore all ports to find https or ssh services
+    for serv in censys_output.get('services', []):
+        if not isinstance(serv, dict):
+            continue
+        if serv.get('service_name').lower() == 'http' and serv.get('certificate', None):
+            try:
+                cert = serv.get('certificate', None)
+                if cert:
+                    # TODO switch to api_v2 once available
+                    # use api_v1 as Certificates endpoint in api_v2 doesn't yet provide all the details
+                    cert_details = CensysCertificates(api_id, api_secret).view(cert)
+                    cert_obj = get_certificate_object(cert_details, attribute)
+                    misp_event.add_object(**cert_obj)
+            except KeyError:
+                print("Error !")
+        if serv.get('ssh') and serv.get('service_name').lower() == 'ssh':
+            try:
+                cert = serv.get('ssh').get('server_host_key').get('fingerprint_sha256')
+                # TODO enable once the type is merged
+                # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256'])
+            except KeyError:
+                pass
+
+    # Info from certificate query
+    if "parsed" in censys_output:
+        cert_obj = get_certificate_object(censys_output, attribute)
+        misp_event.add_object(**cert_obj)
+
+    # Location can be present for IP/Websites results
+    if "location" in censys_output:
+        loc_obj = MISPObject('geolocation')
+        loc = censys_output['location']
+        loc_obj.add_attribute('latitude', value=loc.get('coordinates', {}).get('latitude', None))
+        loc_obj.add_attribute('longitude', value=loc.get('coordinates', {}).get('longitude', None))
+        if 'city' in loc:
+            loc_obj.add_attribute('city', value=loc.get('city'))
+        loc_obj.add_attribute('country', value=loc.get('country'))
+        if 'postal_code' in loc:
+            loc_obj.add_attribute('zipcode', value=loc.get('postal_code'))
+        if 'province' in loc:
+            loc_obj.add_attribute('region', value=loc.get('province'))
+        loc_obj.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**loc_obj)
+
+    event = json.loads(misp_event.to_json())
+    return {'Object': event.get('Object', []), 'Attribute': event.get('Attribute', [])}
+
+
+# In case of multiple enrichment (ip and domain), we need to filter out similar objects
+# TODO: make it more granular
+def remove_duplicates(results):
+    # Only one enrichment was performed so no duplicate
+    if len(results) == 1:
+        return results[0]
+    else:
+        final_result = results[0]
+        for i,result in enumerate(results[1:]):
+            obj_l = results[i+1].get('Object', [])
+            for o2 in obj_l:
+                if o2['name'] == "asn":
+                    key = "asn"
+                elif o2['name'] == "ip-port":
+                    key = "ip"
+                elif o2['name'] == "x509":
+                    key = "x509-fingerprint-sha256"
+                elif o2['name'] == "geolocation":
+                    key = "latitude"
+                if not check_if_present(o2, key, final_result.get('Object', [])):
+                    final_result['Object'].append(o2)
+
+    return final_result
+
+
+def check_if_present(object, attribute_name, list_objects):
+    """
+    Assert if a given object is present in the list.
+
+    This function check if object (json format) is present in list_objects
+    using attribute_name for the matching
+    """
+    for o in list_objects:
+        # We first look for a match on the name
+        if o['name'] == object['name']:
+            for attr in object['Attribute']:
+                # Within the attributes, we look for the one to compare
+                if attr['type'] == attribute_name:
+                    # Then we check the attributes of the other object and look for a match
+                    for attr2 in o['Attribute']:
+                        if attr2['type'] == attribute_name and attr2['value'] == attr['value']:
+                            return True
+
+    return False
+
+
+def get_certificate_object(cert, attribute):
+    parsed = cert['parsed']
+    cert_object = MISPObject('x509')
+    cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256'])
+    cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1'])
+    cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5'])
+    cert_object.add_attribute('serial-number', value=parsed['serial_number'])
+    cert_object.add_attribute('version', value=parsed['version'])
+    cert_object.add_attribute('subject', value=parsed['subject_dn'])
+    cert_object.add_attribute('issuer', value=parsed['issuer_dn'])
+    cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start']))
+    cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end']))
+    cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed'])
+    cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name'])
+
+    cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name'])
+
+    if 'rsa_public_key' in parsed['subject_key_info']:
+        pub_key = parsed['subject_key_info']['rsa_public_key']
+        cert_object.add_attribute('pubkey-info-size', value=pub_key['length'])
+        cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent'])
+        hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode()
+        cert_object.add_attribute('pubkey-info-modulus', value=hex_mod)
+
+    if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]:
+        san = parsed["extensions"]["subject_alt_name"]
+        if "dns_names" in san:
+            for dns in san['dns_names']:
+                cert_object.add_attribute('dns_names', value=dns)
+        if "ip_addresses" in san:
+            for ip in san['ip_addresses']:
+                cert_object.add_attribute('ip', value=ip)
+
+    if "raw" in cert:
+        cert_object.add_attribute('raw-base64', value=cert['raw'])
+
+    cert_object.add_reference(attribute.uuid, 'associated-to')
+    return cert_object
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/circl_passivedns.py

@@ -0,0 +1,85 @@
+import pypdns
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'}
+moduleinfo = {
+    'version': '0.2',
+    'author': 'Alexandre Dulaunoy',
+    'description': 'Module to access CIRCL Passive DNS.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'CIRCL Passive DNS',
+    'logo': 'passivedns.png',
+    'requirements': ['pypdns: Passive DNS python library', 'A CIRCL passive DNS account with username & password'],
+    'features': 'This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects.\n\nTo make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API.',
+    'references': ['https://www.circl.lu/services/passive-dns/', 'https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/'],
+    'input': 'Hostname, domain, or ip-address attribute.',
+    'output': '',
+    'ouput': 'Passive DNS objects related to the input attribute.',
+}
+moduleconfig = ['username', 'password']
+
+
+class PassiveDNSParser:
+    def __init__(self, attribute, authentication):
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.misp_event.add_attribute(**self.attribute)
+        self.pdns = pypdns.PyPDNS(basic_auth=authentication)
+
+    def get_results(self):
+        if hasattr(self, 'result'):
+            return self.result
+        event = self.misp_event.to_dict()
+        results = {key: event[key] for key in ('Attribute', 'Object')}
+        return {'results': results}
+
+    def parse(self):
+        value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value
+
+        try:
+            results = self.pdns.query(value)
+        except Exception:
+            self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
+            return
+
+        if not results:
+            self.result = {'error': 'Not found'}
+            return
+
+        mapping = {'count': 'counter', 'origin': 'text',
+                   'time_first': 'datetime', 'rrtype': 'text',
+                   'rrname': 'text', 'rdata': 'text',
+                   'time_last': 'datetime'}
+        for result in results:
+            pdns_object = MISPObject('passive-dns')
+            for relation, attribute_type in mapping.items():
+                pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
+            pdns_object.add_reference(self.attribute.uuid, 'associated-to')
+            self.misp_event.add_object(**pdns_object)
+
+
+def dict_handler(request: dict):
+    if not request.get('config'):
+        return {'error': 'CIRCL Passive DNS authentication is missing.'}
+    if not request['config'].get('username') or not request['config'].get('password'):
+        return {'error': 'CIRCL Passive DNS authentication is incomplete, please provide your username and password.'}
+    authentication = (request['config']['username'], request['config']['password'])
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
+        return {'error': 'Unsupported attribute type.'}
+    pdns_parser = PassiveDNSParser(attribute, authentication)
+    pdns_parser.parse()
+    return pdns_parser.get_results()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo