misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/expansion/apivoid.py

@@ -0,0 +1,133 @@
+import json
+import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['domain', 'hostname', 'email', 'email-src', 'email-dst', 'email-reply-to', 'dns-soa-email', 'target-email', 'whois-registrant-email'], 'format': 'misp_standard'}
+moduleinfo = {
+    'version': '0.2',
+    'author': 'Christian Studer',
+    'description': 'Module to query APIVoid with some domain attributes.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'APIVoid',
+    'logo': 'apivoid.png',
+    'requirements': ['A valid APIVoid API key with enough credits to proceed 2 queries'],
+    'features': 'This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates. It returns then those pieces of data as MISP objects that can be added to the event.\n\nTo make it work, a valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required.',
+    'references': ['https://www.apivoid.com/'],
+    'input': 'A domain attribute.',
+    'output': 'DNS records and SSL certificates related to the domain.',
+}
+moduleconfig = ['apikey']
+
+
+class APIVoidParser():
+    def __init__(self, attribute):
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.misp_event.add_attribute(**self.attribute)
+        self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
+
+    def get_results(self):
+        if hasattr(self, 'result'):
+            return self.result
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in ('Attribute', 'Object')}
+        return {'results': results}
+
+    def parse_domain(self, apikey):
+        feature = 'dnslookup'
+        if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.13:
+            self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'}
+            return
+        mapping = {'A': 'resolution-of', 'MX': 'mail-server-of', 'NS': 'server-name-of'}
+        dnslookup = requests.get(f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}').json()
+        for item in dnslookup['data']['records']['items']:
+            record_type = item['type']
+            try:
+                relationship = mapping[record_type]
+            except KeyError:
+                continue
+            self._handle_dns_record(item, record_type, relationship)
+        ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json()
+        self._parse_ssl_certificate(ssl['data']['certificate'])
+
+    def handle_email(self, apikey):
+        feature = 'emailverify'
+        if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06:
+            self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'}
+            return
+        emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json()
+        email_verification = MISPObject('apivoid-email-verification')
+        boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username',
+                              'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records',
+                              'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email',
+                              'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain',
+                              'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain',
+                              'should_block']
+        for boolean_attribute in boolean_attributes:
+            email_verification.add_attribute(boolean_attribute,
+                                             **{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]})
+        email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']})
+        email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']})
+        email_verification.add_attribute('role_address',
+                                         **{'type': 'boolean', 'value': emaillookup['data']['role_address']})
+        email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']})
+        email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']})
+        email_verification.add_reference(self.attribute['uuid'], 'related-to')
+        self.misp_event.add_object(email_verification)
+
+    def _handle_dns_record(self, item, record_type, relationship):
+        dns_record = MISPObject('dns-record')
+        dns_record.add_attribute('queried-domain', type='domain', value=item['host'])
+        attribute_type, feature = ('ip-dst', 'ip') if record_type == 'A' else ('domain', 'target')
+        dns_record.add_attribute(f'{record_type.lower()}-record', type=attribute_type, value=item[feature])
+        dns_record.add_reference(self.attribute.uuid, relationship)
+        self.misp_event.add_object(**dns_record)
+
+    def _parse_ssl_certificate(self, certificate):
+        x509 = MISPObject('x509')
+        fingerprint = 'x509-fingerprint-sha1'
+        x509.add_attribute(fingerprint, type=fingerprint, value=certificate['fingerprint'])
+        x509_mapping = {'subject': {'name': ('text', 'subject')},
+                        'issuer': {'common_name': ('text', 'issuer')},
+                        'signature': {'serial': ('text', 'serial-number')},
+                        'validity': {'valid_from': ('datetime', 'validity-not-before'),
+                                     'valid_to': ('datetime', 'validity-not-after')}}
+        certificate = certificate['details']
+        for feature, subfeatures in x509_mapping.items():
+            for subfeature, mapping in subfeatures.items():
+                attribute_type, relation = mapping
+                x509.add_attribute(relation, type=attribute_type, value=certificate[feature][subfeature])
+        x509.add_reference(self.attribute.uuid, 'seen-by')
+        self.misp_event.add_object(**x509)
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('config', {}).get('apikey'):
+        return {'error': 'An API key for APIVoid is required.'}
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+    apikey = request['config']['apikey']
+    apivoid_parser = APIVoidParser(attribute)
+    if attribute['type'] in ['domain', 'hostname']:
+        apivoid_parser.parse_domain(apikey)
+    else:
+        apivoid_parser.handle_email(apikey)
+    return apivoid_parser.get_results()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/assemblyline_query.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+import json
+from . import check_input_attribute, standard_error_message
+from assemblyline_client import Client, ClientError
+from collections import defaultdict
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['link'], 'format': 'misp_standard'}
+
+moduleinfo = {
+    'version': '1',
+    'author': 'Christian Studer',
+    'description': 'A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.',
+    'module-type': ['expansion'],
+    'name': 'AssemblyLine Query',
+    'logo': 'assemblyline.png',
+    'requirements': ['assemblyline_client: Python library to query the AssemblyLine rest API.'],
+    'features': 'The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the used-ID and an API key or the password associated to the user-ID.\n\nThe submission ID extracted from the submission link is then used to query AssemblyLine and get the full submission report. This report is parsed to extract file objects and the associated IPs, domains or URLs the files are connecting to.\n\nSome more data may be parsed in the future.',
+    'references': ['https://www.cyber.gc.ca/en/assemblyline'],
+    'input': 'Link of an AssemblyLine submission report.',
+    'output': 'MISP attributes & objects parsed from the AssemblyLine submission.',
+}
+moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"]
+
+
+class AssemblyLineParser():
+    def __init__(self):
+        self.misp_event = MISPEvent()
+        self.results = {}
+        self.attribute = {'to_ids': True}
+        self._results_mapping = {'NET_DOMAIN_NAME': 'domain', 'NET_FULL_URI': 'url',
+                                 'NET_IP': 'ip-dst'}
+        self._file_mapping = {'entropy': {'type': 'float', 'object_relation': 'entropy'},
+                              'md5': {'type': 'md5', 'object_relation': 'md5'},
+                              'mime': {'type': 'mime-type', 'object_relation': 'mimetype'},
+                              'sha1': {'type': 'sha1', 'object_relation': 'sha1'},
+                              'sha256': {'type': 'sha256', 'object_relation': 'sha256'},
+                              'size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
+                              'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}}
+
+    def get_submission(self, attribute, client):
+        sid = attribute['value'].split('=')[-1]
+        try:
+            if not client.submission.is_completed(sid):
+                self.results['error'] = 'Submission not completed, please try again later.'
+                return
+        except Exception as e:
+            self.results['error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}'
+            return
+        try:
+            submission = client.submission.full(sid)
+        except Exception as e:
+            self.results['error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}"
+            return
+        self._parse_report(submission)
+
+    def finalize_results(self):
+        if 'error' in self.results:
+            return self.results
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key])}
+        return {'results': results}
+
+    def _create_attribute(self, result, attribute_type):
+        attribute = MISPAttribute()
+        attribute.from_dict(type=attribute_type, value=result['value'], **self.attribute)
+        if result['classification'] != 'UNCLASSIFIED':
+            attribute.add_tag(result['classification'].lower())
+        self.misp_event.add_attribute(**attribute)
+        return {'referenced_uuid': attribute.uuid, 'relationship_type': '-'.join(result['context'].lower().split(' '))}
+
+    def _create_file_object(self, file_info):
+        file_object = MISPObject('file')
+        filename_attribute = {'type': 'filename'}
+        filename_attribute.update(self.attribute)
+        if file_info['classification'] != "UNCLASSIFIED":
+            tag = {'Tag': [{'name': file_info['classification'].lower()}]}
+            filename_attribute.update(tag)
+            for feature, attribute in self._file_mapping.items():
+                attribute.update(tag)
+                file_object.add_attribute(value=file_info[feature], **attribute)
+            return filename_attribute, file_object
+        for feature, attribute in self._file_mapping.items():
+            file_object.add_attribute(value=file_info[feature], **attribute)
+        return filename_attribute, file_object
+
+    @staticmethod
+    def _get_results(submission_results):
+        results = defaultdict(list)
+        for k, values in submission_results.items():
+            h = k.split('.')[0]
+            for t in values['result']['tags']:
+                if t['context'] is not None:
+                    results[h].append(t)
+        return results
+
+    def _get_scores(self, file_tree):
+        scores = {}
+        for h, f in file_tree.items():
+            score = f['score']
+            if score > 0:
+                scores[h] = {'name': f['name'], 'score': score}
+            if f['children']:
+                scores.update(self._get_scores(f['children']))
+        return scores
+
+    def _parse_report(self, submission):
+        if submission['classification'] != 'UNCLASSIFIED':
+            self.misp_event.add_tag(submission['classification'].lower())
+        filtered_results = self._get_results(submission['results'])
+        scores = self._get_scores(submission['file_tree'])
+        for h, results in filtered_results.items():
+            if h in scores:
+                attribute, file_object = self._create_file_object(submission['file_infos'][h])
+                print(file_object)
+                for filename in scores[h]['name']:
+                    file_object.add_attribute('filename', value=filename, **attribute)
+                for reference in self._parse_results(results):
+                    file_object.add_reference(**reference)
+                self.misp_event.add_object(**file_object)
+
+    def _parse_results(self, results):
+        references = []
+        for result in results:
+            try:
+                attribute_type = self._results_mapping[result['type']]
+            except KeyError:
+                continue
+            references.append(self._create_attribute(result, attribute_type))
+        return references
+
+
+def parse_config(apiurl, user_id, config):
+    error = {"error": "Please provide your AssemblyLine API key or Password."}
+    if config.get('apikey'):
+        try:
+            return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl'])
+        except ClientError as e:
+            error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}'
+    if config.get('password'):
+        try:
+            return Client(apiurl, auth=(user_id, config['password']))
+        except ClientError as e:
+            error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}'
+    return error
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+    if not request.get('config'):
+        return {"error": "Missing configuration."}
+    if not request['config'].get('apiurl'):
+        return {"error": "No AssemblyLine server address provided."}
+    apiurl = request['config']['apiurl']
+    if not request['config'].get('user_id'):
+        return {"error": "Please provide your AssemblyLine User ID."}
+    user_id = request['config']['user_id']
+    client = parse_config(apiurl, user_id, request['config'])
+    if isinstance(client, dict):
+        return client
+    assemblyline_parser = AssemblyLineParser()
+    assemblyline_parser.get_submission(request['attribute'], client)
+    return assemblyline_parser.finalize_results()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/assemblyline_submit.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import json
+
+from assemblyline_client import Client, ClientError
+from urllib.parse import urljoin
+
+
+moduleinfo = {
+    'version': 1,
+    'author': 'Christian Studer',
+    'module-type': ['expansion'],
+    'name': 'AssemblyLine Submit',
+    'description': 'A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.',
+    'logo': 'assemblyline.png',
+    'requirements': ['assemblyline_client: Python library to query the AssemblyLine rest API.'],
+    'features': 'The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the user-ID and an API key or the password associated to the user-ID.\n\nIf the sample or url is correctly submitted, you get then the link of the submission.',
+    'references': ['https://www.cyber.gc.ca/en/assemblyline'],
+    'input': 'Sample, or url to submit to AssemblyLine.',
+    'output': 'Link of the report generated in AssemblyLine.',
+}
+moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"]
+mispattributes = {"input": ["attachment", "malware-sample", "url"],
+                  "output": ["link"]}
+
+
+def parse_config(apiurl, user_id, config):
+    error = {"error": "Please provide your AssemblyLine API key or Password."}
+    if config.get('apikey'):
+        try:
+            return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl'])
+        except ClientError as e:
+            error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}'
+    if config.get('password'):
+        try:
+            return Client(apiurl, auth=(user_id, config['password']), verify=config['verifyssl'])
+        except ClientError as e:
+            error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}'
+    return error
+
+
+def submit_content(client, filename, data):
+    try:
+        return client.submit(fname=filename, contents=data.encode())
+    except Exception as e:
+        return {'error': f'Error while submitting content to AssemblyLine: {e.__str__()}'}
+
+
+def submit_request(client, request):
+    if 'attachment' in request:
+        return submit_content(client, request['attachment'], request['data'])
+    if 'malware-sample' in request:
+        return submit_content(client, request['malware-sample'].split('|')[0], request['data'])
+    for feature in ('url', 'domain'):
+        if feature in request:
+            return submit_url(client, request[feature])
+    return {"error": "No valid attribute type for this module has been provided."}
+
+
+def submit_url(client, url):
+    try:
+        return client.submit(url=url)
+    except Exception as e:
+        return {'error': f'Error while submitting url to AssemblyLine: {e.__str__()}'}
+
+
+def handler(q=False):
+    if q is False:
+        return q
+    request = json.loads(q)
+    if not request.get('config'):
+        return {"error": "Missing configuration."}
+    if not request['config'].get('apiurl'):
+        return {"error": "No AssemblyLine server address provided."}
+    apiurl = request['config']['apiurl']
+    if not request['config'].get('user_id'):
+        return {"error": "Please provide your AssemblyLine User ID."}
+    user_id = request['config']['user_id']
+    client = parse_config(apiurl, user_id, request['config'])
+    if isinstance(client, dict):
+        return client
+    submission = submit_request(client, request)
+    if 'error' in submission:
+        return submission
+    sid = submission['submission']['sid']
+    return {
+        "results": [{
+            "types": "link",
+            "categories": "External analysis",
+            "values": urljoin(apiurl, f'submission_detail.html?sid={sid}')
+        }]
+    }
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo["config"] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/backscatter_io.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""Backscatter.io Module."""
+import json
+try:
+    from backscatter import Backscatter
+except ImportError:
+    print("Backscatter.io library not installed.")
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
+moduleinfo = {
+    'version': '1',
+    'author': 'brandon@backscatter.io',
+    'description': 'Backscatter.io module to bring mass-scanning observations into MISP.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'Backscatter.io',
+    'logo': 'backscatter_io.png',
+    'requirements': ['backscatter python library'],
+    'features': 'The module takes a source or destination IP address as input and displays the information known by backscatter.io.',
+    'references': ['https://pypi.org/project/backscatter/'],
+    'input': 'IP addresses.',
+    'output': 'Text containing a history of the IP addresses especially on scanning based on backscatter.io information .',
+}
+moduleconfig = ['api_key']
+query_playbook = [
+    {'inputs': ['ip-src', 'ip-dst'],
+     'services': ['observations', 'enrichment'],
+     'name': 'generic'}
+]
+
+
+def check_query(request):
+    """Check the incoming request for a valid configuration."""
+    output = {'success': False}
+    config = request.get('config', None)
+    if not config:
+        misperrors['error'] = "Configuration is missing from the request."
+        return output
+    for item in moduleconfig:
+        if config.get(item, None):
+            continue
+        misperrors['error'] = "Backscatter.io authentication is missing."
+        return output
+    if not request.get('ip-src') and request.get('ip-dst'):
+        misperrors['error'] = "Unsupported attributes type."
+        return output
+    profile = {'success': True, 'config': config, 'playbook': 'generic'}
+    if 'ip-src' in request:
+        profile.update({'value': request.get('ip-src')})
+    else:
+        profile.update({'value': request.get('ip-dst')})
+    return profile
+
+
+def handler(q=False):
+    """Handle gathering data."""
+    if not q:
+        return q
+    request = json.loads(q)
+    checks = check_query(request)
+    if not checks['success']:
+        return misperrors
+
+    try:
+        bs = Backscatter(checks['config']['api_key'])
+        response = bs.get_observations(query=checks['value'], query_type='ip')
+        if not response['success']:
+            misperrors['error'] = '%s: %s' % (response['error'], response['message'])
+            return misperrors
+        output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]}
+    except Exception as e:
+        misperrors['error'] = str(e)
+        return misperrors
+
+    return output
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/btc_scam_check.py

@@ -0,0 +1,54 @@
+import json
+import sys
+
+try:
+    from dns.resolver import Resolver, NXDOMAIN
+    from dns.name import LabelTooLong
+    resolver = Resolver()
+    resolver.timeout = 1
+    resolver.lifetime = 1
+except ImportError:
+    sys.exit("dnspython3 in missing. use 'pip install dnspython3' to install it.")
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['btc'], 'output': ['text']}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Christian Studer',
+    'description': 'An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.',
+    'module-type': ['hover'],
+    'name': 'BTC Scam Check',
+    'logo': 'bitcoin.png',
+    'requirements': ['dnspython3: dns python library'],
+    'features': 'The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.',
+    'references': ['https://btcblack.it/'],
+    'input': 'btc address attribute.',
+    'output': 'Text to indicate if the BTC address has been abused.',
+}
+moduleconfig = []
+
+url = 'bl.btcblack.it'
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    btc = request['btc']
+    query = f"{btc}.{url}"
+    try:
+        result = ' - '.join([str(r) for r in resolver.resolve(query, 'TXT')])[1:-1]
+    except NXDOMAIN:
+        result = f"{btc} is not known as a scam address."
+    except LabelTooLong:
+        result = f"{btc} is probably not a valid BTC address."
+    return {'results': [{'types': mispattributes['output'], 'values': result}]}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo