misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/expansion/variotdbs.py

@@ -0,0 +1,133 @@
+import json
+import requests
+from . import check_input_attribute, standard_error_message
+from ._vulnerability_parser.vulnerability_parser import (
+    VulnerabilityMapping, VulnerabilityParser)
+from pymisp import MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'}
+moduleinfo = {
+    'version': '1',
+    'author': 'Christian Studer',
+    'description': 'An expansion module to query the VARIoT db API for more information about a vulnerability.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'VARIoT db Lookup',
+    'logo': 'variot.png',
+    'requirements': ['A VARIoT db API key (if you do not want to be limited to 100 queries / day)'],
+    'features': 'The module takes a vulnerability attribute as input and queries que VARIoT db API to gather additional information.\n\nThe `vuln` endpoint is queried first to look for additional information about the vulnerability itself.\n\nThe `exploits` endpoint is also queried then to look for the information of the potential related exploits, which are parsed and added to the results using the `exploit` object template.',
+    'references': ['https://www.variotdbs.pl/'],
+    'input': 'Vulnerability attribute.',
+    'output': 'Additional information about the vulnerability, as it is stored on the VARIoT db, about the vulnerability itself, and the potential related exploits.',
+}
+moduleconfig = ['API_key']
+variotdbs_url = 'https://www.variotdbs.pl/api'
+
+
+class VariotMapping(VulnerabilityMapping):
+    __exploit_mapping = {
+        'credits': 'credit',
+        'description': 'description',
+        'exploit': 'exploit',
+        'title': 'title'
+    }
+    __exploit_multiple_mapping = {
+        'cve': {
+            'feature': 'cve_id',
+            'relation': 'cve-id'
+        },
+        'references': {
+            'feature': 'url',
+            'relation': 'reference'
+        }
+    }
+
+    @classmethod
+    def exploit_mapping(cls) -> dict:
+        return cls.__exploit_mapping
+
+    @classmethod
+    def exploit_multiple_mapping(cls) -> dict:
+        return cls.__exploit_multiple_mapping
+
+
+class VariotdbsParser(VulnerabilityParser):
+    def __init__(self, attribute):
+        super().__init__(attribute)
+        self.__mapping = VulnerabilityMapping
+
+    @property
+    def mapping(self) -> VulnerabilityMapping:
+        return self.__mapping
+
+    def parse_exploit_information(self, query_results):
+        for exploit in query_results:
+            exploit_object = MISPObject('exploit')
+            exploit_object.add_attribute('exploitdb-id', exploit['edb_id'])
+            for field, relation in self.mapping.exploit_mapping().items():
+                if exploit.get(field):
+                    exploit_object.add_attribute(
+                        relation, exploit[field]['data']
+                    )
+            for field, relation in self.mapping.exploit_multiple_mapping().items():
+                if exploit.get(field):
+                    for value in exploit[field]['data']:
+                        exploit_object.add_attribute(
+                            relation['relation'], value[relation['feature']]
+                        )
+            exploit_object.add_reference(self.misp_attribute.uuid, 'related-to')
+            self.misp_event.add_object(exploit_object)
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute.get('type') != 'vulnerability':
+        return {'error': 'Vulnerability id missing.'}
+    headers = {'Content-Type': 'application/json'}
+    if request.get('config', {}).get('API_key'):
+        headers['Authorization'] = f"Token {request['config']['API_key']}"
+    empty = True
+    parser = VariotdbsParser(attribute)
+    r = requests.get(f"{variotdbs_url}/vuln/{attribute['value']}/", headers=headers)
+    if r.status_code == 200:
+        vulnerability_results = r.json()
+        if vulnerability_results:
+            parser._parse_variot_description(vulnerability_results)
+            empty = False
+    else:
+        if r.reason != 'Not Found':
+            return {'error': 'Error while querying the variotdbs API.'}
+    r = requests.get(f"{variotdbs_url}/exploits/?cve={attribute['value']}", headers=headers)
+    if r.status_code == 200:
+        exploit_results = r.json()
+        if exploit_results:
+            parser.parse_exploit_information(exploit_results['results'])
+            empty = False
+            if exploit_results['next'] is not None:
+                while(1):
+                    exploit_results = requests.get(exploit_results['next'], headers=headers)
+                    if exploit_results.status_code != 200:
+                        break
+                    exploit_results = exploit_results.json()
+                    parser.parse_exploit_information(exploit_results['results'])
+                    if exploit_results['next'] is None:
+                        break
+    else:
+        return {'error': 'Error while querying the variotdbs API.'}
+    if empty:
+        return {'error': 'Empty results'}
+    return parser.get_results()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/virustotal.py

@@ -0,0 +1,311 @@
+from urllib.parse import urlparse
+import vt
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url", "ip-src|port", "ip-dst|port"],
+                  'format': 'misp_standard'}
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {
+    'version': '6',
+    'author': 'Hannah Ward',
+    'description': 'Enrich observables with the VirusTotal v3 API',
+    'module-type': ['expansion'],
+    'name': 'VirusTotal v3 Lookup',
+    'logo': 'virustotal.png',
+    'requirements': ['An access to the VirusTotal API (apikey), with a high request rate limit.'],
+    'features': 'New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them.',
+    'references': ['https://www.virustotal.com/', 'https://docs.virustotal.com/reference/overview'],
+    'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.',
+    'output': 'MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.',
+}
+
+# config fields that your code expects from the site admin
+moduleconfig = ["apikey", "event_limit", 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
+
+
+DEFAULT_RESULTS_LIMIT = 10
+
+
+class VirusTotalParser:
+    def __init__(self, client: vt.Client, limit: int) -> None:
+        self.client = client
+        self.limit = limit or DEFAULT_RESULTS_LIMIT
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.parsed_objects = {}
+        self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
+                                    'domain': self.parse_domain, 'hostname': self.parse_domain,
+                                    'md5': self.parse_hash, 'sha1': self.parse_hash,
+                                    'sha256': self.parse_hash, 'url': self.parse_url,
+                                    'ip-src|port': self.parse_ip_port, 'ip-dst|port': self.parse_ip_port}
+        self.proxies = None
+
+    @staticmethod
+    def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int:
+        if not analysis:
+            return 0
+        count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']])
+        return count if known_distributors else count + analysis['malicious']
+
+    def query_api(self, attribute: dict) -> None:
+        self.attribute.from_dict(**attribute)
+        self.input_types_mapping[self.attribute.type](self.attribute.value)
+
+    def get_result(self) -> dict:
+        event = self.misp_event.to_dict()
+        results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+        return {'results': results}
+
+    def add_vt_report(self, report: vt.Object) -> str:
+        analysis = report.get('last_analysis_stats')
+        total = self.get_total_analysis(analysis, report.get('known_distributors'))
+        if report.type == 'ip_address':
+            rtype = 'ip-address'
+        else:
+            rtype = report.type
+        permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}'
+
+        vt_object = MISPObject('virustotal-report')
+        vt_object.add_attribute('permalink', type='link', value=permalink)
+        detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-'
+        vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True)
+        self.misp_event.add_object(**vt_object)
+        return vt_object.uuid
+
+    def create_misp_object(self, report: vt.Object) -> MISPObject:
+        misp_object = None
+        vt_uuid = self.add_vt_report(report)
+
+        if report.type == 'file':
+            misp_object = MISPObject('file')
+            for hash_type in ('md5', 'sha1', 'sha256', 'tlsh',
+                              'vhash', 'ssdeep', 'imphash'):
+                misp_object.add_attribute(hash_type,
+                                          **{'type': hash_type,
+                                             'value': report.get(hash_type)})
+        elif report.type == 'domain':
+            misp_object = MISPObject('domain-ip')
+            misp_object.add_attribute('domain', type='domain', value=report.id)
+        elif report.type == 'ip_address':
+            misp_object = MISPObject('domain-ip')
+            misp_object.add_attribute('ip', type='ip-dst', value=report.id)
+        elif report.type == 'url':
+            misp_object = MISPObject('url')
+            misp_object.add_attribute('url', type='url', value=report.id)
+        misp_object.add_reference(vt_uuid, 'analyzed-with')
+        return misp_object
+
+    ################################################################################
+    ####                         Main parsing functions                         #### # noqa
+    ################################################################################
+
+    def parse_domain(self, domain: str) -> str:
+        domain_report = self.client.get_object(f'/domains/{domain}')
+
+        # DOMAIN
+        domain_object = self.create_misp_object(domain_report)
+
+        # WHOIS
+        if domain_report.whois:
+            whois_object = MISPObject('whois')
+            whois_object.add_attribute('text', type='text', value=domain_report.whois)
+            self.misp_event.add_object(**whois_object)
+
+        # SIBLINGS AND SUBDOMAINS
+        for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]:
+            rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
+            for item in rel_iterator:
+                attr = MISPAttribute()
+                attr.from_dict(**dict(type='domain', value=item.id))
+                self.misp_event.add_attribute(**attr)
+                domain_object.add_reference(attr.uuid, misp_name)
+
+        # RESOLUTIONS
+        resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit)
+        for resolution in resolutions_iterator:
+            domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address)
+
+        # COMMUNICATING, DOWNLOADED AND REFERRER FILES
+        for relationship_name, misp_name in [
+            ('communicating_files', 'communicates-with'),
+            ('downloaded_files', 'downloaded-from'),
+            ('referrer_files', 'referring')
+        ]:
+            files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
+            for file in files_iterator:
+                file_object = self.create_misp_object(file)
+                file_object.add_reference(domain_object.uuid, misp_name)
+                self.misp_event.add_object(**file_object)
+
+        # URLS
+        urls_iterator = self.client.iterator(f'/domains/{domain_report.id}/urls', limit=self.limit)
+        for url in urls_iterator:
+            url_object = self.create_misp_object(url)
+            url_object.add_reference(domain_object.uuid, 'hosted-in')
+            self.misp_event.add_object(**url_object)
+
+        self.misp_event.add_object(**domain_object)
+        return domain_object.uuid
+
+    def parse_hash(self, file_hash: str) -> str:
+        file_report = self.client.get_object(f'/files/{file_hash}')
+        file_object = self.create_misp_object(file_report)
+
+        # ITW URLS
+        urls_iterator = self.client.iterator(f'/files/{file_report.id}/itw_urls', limit=self.limit)
+        for url in urls_iterator:
+            url_object = self.create_misp_object(url)
+            url_object.add_reference(file_object.uuid, 'downloaded')
+            self.misp_event.add_object(**url_object)
+
+        # COMMUNICATING, DOWNLOADED AND REFERRER FILES
+        for relationship_name, misp_name in [
+            ('contacted_urls', 'communicates-with'),
+            ('contacted_domains', 'communicates-with'),
+            ('contacted_ips', 'communicates-with')
+        ]:
+            related_files_iterator = self.client.iterator(f'/files/{file_report.id}/{relationship_name}', limit=self.limit)
+            for related_file in related_files_iterator:
+                related_file_object = self.create_misp_object(related_file)
+                related_file_object.add_reference(file_object.uuid, misp_name)
+                self.misp_event.add_object(**related_file_object)
+
+        self.misp_event.add_object(**file_object)
+        return file_object.uuid
+    def parse_ip_port(self, ipport: str) -> str:
+        ip = ipport.split('|')[0]
+        self.parse_ip(ip)
+
+    def parse_ip(self, ip: str) -> str:
+        ip_report = self.client.get_object(f'/ip_addresses/{ip}')
+
+        # IP
+        ip_object = self.create_misp_object(ip_report)
+
+        # ASN
+        asn_object = MISPObject('asn')
+        asn_object.add_attribute('asn', type='AS', value=ip_report.asn)
+        asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network)
+        asn_object.add_attribute('country', type='text', value=ip_report.country)
+        self.misp_event.add_object(**asn_object)
+
+        # RESOLUTIONS
+        resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit)
+        for resolution in resolutions_iterator:
+            ip_object.add_attribute('domain', type='domain', value=resolution.host_name)
+
+        # URLS
+        urls_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/urls', limit=self.limit)
+        for url in urls_iterator:
+            url_object = self.create_misp_object(url)
+            url_object.add_reference(ip_object.uuid, 'hosted-in')
+            self.misp_event.add_object(**url_object)
+
+        self.misp_event.add_object(**ip_object)
+        return ip_object.uuid
+
+    def parse_url(self, url: str) -> str:
+        url_id = vt.url_id(url)
+        url_report = self.client.get_object(f'/urls/{url_id}')
+        url_object = self.create_misp_object(url_report)
+
+        # COMMUNICATING, DOWNLOADED AND REFERRER FILES
+        for relationship_name, misp_name in [
+            ('communicating_files', 'communicates-with'),
+            ('downloaded_files', 'downloaded-from'),
+            ('referrer_files', 'referring')
+        ]:
+            files_iterator = self.client.iterator(f'/urls/{url_report.id}/{relationship_name}', limit=self.limit)
+            for file in files_iterator:
+                file_object = self.create_misp_object(file)
+                file_object.add_reference(url_object.uuid, misp_name)
+                self.misp_event.add_object(**file_object)
+
+        self.misp_event.add_object(**url_object)
+        return url_object.uuid
+
+
+def get_proxy_settings(config: dict) -> dict:
+    """Returns proxy settings in the requests format.
+    If no proxy settings are set, return None."""
+    proxies = None
+    host = config.get('proxy_host')
+    port = config.get('proxy_port')
+    username = config.get('proxy_username')
+    password = config.get('proxy_password')
+
+    if host:
+        if not port:
+            misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
+                                  'please also set the virustotal_proxy_port.'
+            raise KeyError
+        parsed = urlparse(host)
+        if 'http' in parsed.scheme:
+            scheme = 'http'
+        else:
+            scheme = parsed.scheme
+        netloc = parsed.netloc
+        host = f'{netloc}:{port}'
+
+        if username:
+            if not password:
+                misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
+                                      'please also set the virustotal_proxy_password.'
+                raise KeyError
+            auth = f'{username}:{password}'
+            host = auth + '@' + host
+
+        proxies = {
+            'http': f'{scheme}://{host}',
+            'https': f'{scheme}://{host}'
+        }
+    return proxies
+
+
+def parse_error(status_code: int) -> str:
+    status_mapping = {204: 'VirusTotal request rate limit exceeded.',
+                      400: 'Incorrect request, please check the arguments.',
+                      403: 'You don\'t have enough privileges to make the request.'}
+    if status_code in status_mapping:
+        return status_mapping[status_code]
+    return "VirusTotal may not be accessible."
+
+
+def dict_handler(request: dict):
+    if not request.get('config') or not request['config'].get('apikey'):
+        misperrors['error'] = 'A VirusTotal api key is required for this module.'
+        return misperrors
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+
+    event_limit = request['config'].get('event_limit')
+    attribute = request['attribute']
+    proxy_settings = get_proxy_settings(request.get('config'))
+
+    try:
+        client = vt.Client(request['config']['apikey'],
+                           headers={
+                               'x-tool': 'MISPModuleVirusTotalExpansion',
+                           },
+                           proxy=proxy_settings['http'] if proxy_settings else None)
+        parser = VirusTotalParser(client, int(event_limit) if event_limit else None)
+        parser.query_api(attribute)
+    except vt.APIError as ex:
+        misperrors['error'] = ex.message
+        return misperrors
+
+    return parser.get_result()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo