misp-modules 2.4.196__py3-none-any.whl

misp_modules/modules/expansion/greynoise.py

@@ -0,0 +1,346 @@
+import ipaddress
+import json
+import logging
+
+
+try:
+    from greynoise import GreyNoise
+except ImportError:
+    print("greynoise module not installed.")
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+from . import check_input_attribute, standard_error_message
+
+logger = logging.getLogger("greynoise")
+logger.setLevel(logging.INFO)
+
+misperrors = {"error": "Error"}
+mispattributes = {"input": ["ip-src", "ip-dst", "vulnerability"], "format": "misp_standard"}
+moduleinfo = {
+    'version': '1.2',
+    'author': 'Brad Chiappetta <brad@greynoise.io>',
+    'description': 'Module to query IP and CVE information from GreyNoise',
+    'module-type': ['expansion', 'hover'],
+    'name': 'GreyNoise Lookup',
+    'logo': 'greynoise.png',
+    'requirements': ['A Greynoise API key. Both Enterprise (Paid) and Community (Free) API keys are supported, however Community API users will only be able to perform IP lookups.'],
+    'features': 'This module supports: 1) Query an IP from GreyNoise to see if it is internet background noise or a common business service 2) Query a CVE from GreyNoise to see the total number of internet scanners looking for the CVE in the last 7 days.',
+    'references': ['https://greynoise.io/', 'https://docs.greyniose.io/', 'https://www.greynoise.io/viz/account/'],
+    'input': 'An IP address or CVE ID',
+    'output': 'IP Lookup information or CVE scanning profile for past 7 days',
+}
+moduleconfig = ["api_key", "api_type"]
+
+
+class GreyNoiseParser:
+    def __init__(self, attribute):
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.misp_event.add_attribute(**self.attribute)
+        self.ip_address_enrich_mapping = {
+            "noise": {"type": "boolean", "object_relation": "noise"},
+            "riot": {"type": "boolean", "object_relation": "riot"},
+            "classification": {"type": "text", "object_relation": "classification"},
+            "actor": {"type": "text", "object_relation": "actor"},
+            "trust_level": {"type": "text", "object_relation": "trust-level"},
+            "name": {"type": "text", "object_relation": "provider"},
+            "first_seen": {"type": "datetime", "object_relation": "first-seen"},
+            "last_seen": {"type": "datetime", "object_relation": "last-seen"},
+            "link": {"type": "url", "object_relation": "link"},
+            "last_updated": {"type": "datetime", "object_relation": "last-seen"},
+        }
+        self.ip_address_hover_mapping = {
+            "noise": {"type": "boolean", "object_relation": "noise"},
+            "riot": {"type": "boolean", "object_relation": "riot"},
+            "classification": {"type": "text", "object_relation": "classification"},
+            "actor": {"type": "text", "object_relation": "actor"},
+            "tags": {"type": "text", "object_relation": "tags"},
+            "cve": {"type": "text", "object_relation": "cve"},
+            "vpn": {"type": "text", "object_relation": "vpn"},
+            "vpn_service": {"type": "text", "object_relation": "vpn_service"},
+            "bot": {"type": "text", "object_relation": "bot"},
+            "first_seen": {"type": "datetime", "object_relation": "first-seen"},
+            "last_seen": {"type": "datetime", "object_relation": "last-seen"},
+            "spoofable": {"type": "datetime", "object_relation": "spoofable"},
+            "link": {"type": "url", "object_relation": "link"},
+            "category": {"type": "text", "object_relation": "category"},
+            "name": {"type": "text", "object_relation": "provider"},
+            "trust_level": {"type": "text", "object_relation": "trust-level"},
+            "last_updated": {"type": "datetime", "object_relation": "last_updated"},
+        }
+        self.ip_address_metadata_mapping = {
+            "tor": {"type": "text", "object_relation": "tor"},
+            "asn": {"type": "AS", "object_relation": "asn"},
+            "city": {"type": "text", "object_relation": "city"},
+            "country_code": {"type": "text", "object_relation": "country-code"},
+            "country": {"type": "text", "object_relation": "country"},
+            "organization": {"type": "text", "object_relation": "organization"},
+            "destination_country_codes": {"type": "text", "object_relation": "destination-country-codes"},
+            "destination_countries": {"type": "text", "object_relation": "destination-countries"},
+            "category": {"type": "text", "object_relation": "category"},
+            "rdns": {"type": "text", "object_relation": "rdns"},
+        }
+        self.vulnerability_mapping = {
+            "id": {"type": "text", "object_relation": "id"},
+            "details": {"type": "text", "object_relation": "details"},
+            "count": {"type": "text", "object_relation": "total-count"},
+            "benign": {"type": "text", "object_relation": "benign-count"},
+            "malicious": {"type": "text", "object_relation": "malicious-count"},
+            "unknown": {"type": "text", "object_relation": "unknown-count"},
+        }
+
+    def query_greynoise_ip_hover(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting hover enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            quick_response = session.quick(self.attribute.value)
+            if len(quick_response) != 1:
+                misperrors["error"] = "Quick IP lookup returned unexpected response"
+                return misperrors
+            else:
+                quick_response = quick_response[0]
+            context_response = session.ip(self.attribute.value)
+            riot_response = session.riot(self.attribute.value)
+
+            if riot_response and "trust_level" in riot_response:
+                if riot_response["trust_level"] == "1":
+                    riot_response["trust_level"] = "1 - Reasonably Ignore"
+                if riot_response["trust_level"] == "2":
+                    riot_response["trust_level"] = "2 - Commonly Seen"
+
+            if context_response and riot_response:
+                response = context_response.copy()
+                response.update(riot_response)
+                response.update(quick_response)
+            elif context_response:
+                response = context_response.copy()
+                response.update(quick_response)
+            elif riot_response:
+                response = riot_response.copy()
+                response.update(quick_response)
+
+            response["link"] = "https://viz.greynoise.io/ip/" + self.attribute.value
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_hover_mapping.items():
+                logger.debug(f"Checking feature {feature}")
+                if response.get(feature):
+                    if feature in ["cve", "tags"]:
+                        response[feature] = ", ".join(response[feature])
+                    if feature == "vpn_service" and response[feature] == "N/A":
+                        continue
+                    if feature == "actor" and response[feature] == "unknown":
+                        continue
+                    attribute = {"value": response[feature]}
+                    logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if "metadata" in context_response:
+                for feature, mapping in self.ip_address_metadata_mapping.items():
+                    logger.debug(f"Checking metadata feature {feature}")
+                    if response["metadata"].get(feature):
+                        if feature in ["destination_countries", "destination_country_codes"]:
+                            response["metadata"][feature] = ", ".join(response["metadata"][feature])
+                        attribute = {"value": response["metadata"][feature]}
+                        logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                        attribute.update(mapping)
+                        ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                logger.debug("creating greynoise ip object")
+                gn_ip_object = MISPObject("greynoise-ip-details")
+                for attribute in ip_address_attributes:
+                    logger.debug(f"adding attribute {attribute}")
+                    gn_ip_object.add_attribute(**attribute)
+                logger.debug(f"attribute id: {self.attribute.uuid}")
+                gn_ip_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(gn_ip_object)
+        else:
+            logger.info(f"Starting hover enrichment for: {self.attribute.value} via GreyNoise Community API")
+            integration_name = "greynoise-community-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name, offering="community")
+            community_response = session.ip(self.attribute.value)
+
+            if "noise" in community_response and community_response["noise"]:
+                community_response["actor"] = community_response["name"]
+                community_response.pop("name")
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_hover_mapping.items():
+                if community_response.get(feature):
+                    if feature == "actor" and community_response[feature] == "unknown":
+                        continue
+                    attribute = {"value": community_response[feature]}
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                ip_address_object = MISPObject("greynoise-ip-details")
+                for attribute in ip_address_attributes:
+                    ip_address_object.add_attribute(**attribute)
+                ip_address_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(ip_address_object)
+
+    def query_greynoise_ip_expansion(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            quick_response = session.quick(self.attribute.value)
+            if len(quick_response) != 1:
+                misperrors["error"] = "Quick IP lookup returned unexpected response"
+                return misperrors
+            else:
+                quick_response = quick_response[0]
+            context_response = session.ip(self.attribute.value)
+            riot_response = session.riot(self.attribute.value)
+
+            if riot_response and "trust_level" in riot_response:
+                if riot_response["trust_level"] == "1":
+                    riot_response["trust_level"] = "1 - Reasonably Ignore"
+                if riot_response["trust_level"] == "2":
+                    riot_response["trust_level"] = "2 - Commonly Seen"
+
+            if context_response and riot_response:
+                response = context_response.copy()
+                response.update(riot_response)
+                response.update(quick_response)
+            elif context_response:
+                response = context_response.copy()
+                response.update(quick_response)
+            elif riot_response:
+                response = riot_response.copy()
+                response.update(quick_response)
+
+            response["link"] = "https://viz.greynoise.io/ip/" + self.attribute.value
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_enrich_mapping.items():
+                logger.debug(f"Checking feature {feature}")
+                if response.get(feature):
+                    if feature == "actor" and response[feature] == "unknown":
+                        continue
+                    attribute = {"value": response[feature]}
+                    logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                logger.debug("creating greynoise ip object")
+                gn_ip_object = MISPObject("greynoise-ip")
+                for attribute in ip_address_attributes:
+                    logger.debug(f"adding attribute {attribute}")
+                    gn_ip_object.add_attribute(**attribute)
+                logger.debug(f"attribute id: {self.attribute.uuid}")
+                gn_ip_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(gn_ip_object)
+        else:
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise Community API")
+            integration_name = "greynoise-community-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name, offering="community")
+            community_response = session.ip(self.attribute.value)
+
+            if "noise" in community_response and community_response["noise"]:
+                community_response["actor"] = community_response["name"]
+                community_response.pop("name")
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_enrich_mapping.items():
+                if community_response.get(feature):
+                    if feature == "actor" and community_response[feature] == "unknown":
+                        continue
+                    attribute = {"value": community_response[feature]}
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                ip_address_object = MISPObject("greynoise-ip")
+                for attribute in ip_address_attributes:
+                    ip_address_object.add_attribute(**attribute)
+                ip_address_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(ip_address_object)
+
+    def query_greynoise_vulnerability(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            querystring = f"last_seen:1w cve:{self.attribute.value}"
+        else:
+            misperrors["error"] = "Vulnerability Not Supported with Community API Key"
+            return misperrors
+
+        response = session.stats(querystring)
+
+        if "stats" in response:
+            response["details"] = (
+                "The IP count below reflects the number of IPs seen "
+                "by GreyNoise in the last 7 days scanning for this CVE."
+            )
+            response["id"] = self.attribute.value
+            classifications = response["stats"].get("classifications")
+            for item in classifications:
+                if item["classification"] == "benign":
+                    value = item["count"]
+                    response["benign"] = value
+                if item["classification"] == "unknown":
+                    value = item["count"]
+                    response["unknown"] = value
+                if item["classification"] == "malicious":
+                    value = item["count"]
+                    response["malicious"] = value
+            vulnerability_attributes = []
+            for feature, mapping in self.vulnerability_mapping.items():
+                if response.get(feature):
+                    attribute = {"value": response[feature]}
+                    attribute.update(mapping)
+                    vulnerability_attributes.append(attribute)
+            if vulnerability_attributes:
+                vulnerability_object = MISPObject("greynoise-vuln-info")
+                for attribute in vulnerability_attributes:
+                    vulnerability_object.add_attribute(**attribute)
+                vulnerability_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(vulnerability_object)
+
+    def get_result(self):
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
+        return {"results": results}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get("config", {}).get("api_key"):
+        return {"error": "GreyNoise API Key required, but missing"}
+    if not request.get("config", {}).get("api_type"):
+        return {"error": "GreyNoise API type of enterprise or community required, but missing"}
+    if not request.get("attribute") or not check_input_attribute(request["attribute"]):
+        return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."}
+    attribute = request["attribute"]
+    if attribute["type"] not in mispattributes["input"]:
+        return {"error": "Unsupported attribute type."}
+    greynoise_parser = GreyNoiseParser(attribute)
+
+    if attribute["type"] in ["ip-dst", "ip-src"]:
+        try:
+            ipaddress.IPv4Address(attribute["value"])
+            if "persistent" in request:
+                greynoise_parser.query_greynoise_ip_hover(request["config"]["api_key"], request["config"]["api_type"])
+            else:
+                greynoise_parser.query_greynoise_ip_expansion(request["config"]["api_key"], request["config"]["api_type"])
+        except ValueError:
+            return {"error": "Not a valid IPv4 address"}
+
+    if attribute["type"] == "vulnerability":
+        greynoise_parser.query_greynoise_vulnerability(request["config"]["api_key"], request["config"]["api_type"])
+
+    return greynoise_parser.get_result()
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo["config"] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/hashdd.py

@@ -0,0 +1,53 @@
+import json
+import requests
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['md5'], 'output': ['text']}
+moduleinfo = {
+    'version': '0.2',
+    'author': 'Alexandre Dulaunoy',
+    'description': 'A hover module to check hashes against hashdd.com including NSLR dataset.',
+    'module-type': ['hover'],
+    'name': 'Hashdd Lookup',
+    'logo': '',
+    'requirements': [],
+    'features': 'This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed.',
+    'references': ['https://hashdd.com/'],
+    'input': 'A hash MISP attribute (md5).',
+    'output': 'Text describing the known level of the hash in the hashdd databases.',
+}
+moduleconfig = []
+hashddapi_url = 'https://api.hashdd.com/v1/knownlevel/nsrl/'
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    v = None
+    request = json.loads(q)
+    for input_type in mispattributes['input']:
+        if request.get(input_type):
+            v = request[input_type].upper()
+            break
+    if v is None:
+        misperrors['error'] = 'Hash value is missing.'
+        return misperrors
+    r = requests.get(hashddapi_url + v)
+    if r.status_code == 200:
+        state = json.loads(r.text)
+        summary = state['knownlevel'] if state and state['result'] == "SUCCESS" else state['message']
+    else:
+        misperrors['error'] = '{} API not accessible'.format(hashddapi_url)
+        return misperrors['error']
+
+    r = {'results': [{'types': mispattributes['output'], 'values': summary}]}
+    return r
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/hashlookup.py

@@ -0,0 +1,118 @@
+import json
+import requests
+from . import check_input_attribute, standard_error_message
+from collections import defaultdict
+from pymisp import MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['md5', 'sha1', 'sha256'], 'format': 'misp_standard'}
+moduleinfo = {
+    'version': '2',
+    'author': 'Alexandre Dulaunoy',
+    'description': 'An expansion module to query the CIRCL hashlookup services to find it if a hash is part of a known set such as NSRL.',
+    'module-type': ['expansion', 'hover'],
+    'name': 'CIRCL Hashlookup Lookup',
+    'logo': 'circl.png',
+    'requirements': [],
+    'features': 'The module takes file hashes as input such as a MD5 or SHA1.\n It queries the public CIRCL.lu hashlookup service and return all the hits if the hashes are known in an existing dataset. The module can be configured with a custom hashlookup url if required.\n The module can be used an hover module but also an expansion model to add related MISP objects.\n',
+    'references': ['https://www.circl.lu/services/hashlookup/'],
+    'input': 'File hashes (MD5, SHA1)',
+    'output': 'Object with the filename associated hashes if the hash is part of a known set.',
+}
+moduleconfig = ["custom_API"]
+hashlookup_url = 'https://hashlookup.circl.lu/'
+
+
+class HashlookupParser():
+    def __init__(self, attribute, hashlookupresult, api_url):
+        self.attribute = attribute
+        self.hashlookupresult = hashlookupresult
+        self.api_url = api_url
+        self.misp_event = MISPEvent()
+        self.misp_event.add_attribute(**attribute)
+        self.references = defaultdict(list)
+
+    def get_result(self):
+        if self.references:
+            self.__build_references()
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+        return {'results': results}
+
+    def parse_hashlookup_information(self):
+        hashlookup_object = MISPObject('hashlookup')
+        if 'source' in self.hashlookupresult:
+            hashlookup_object.add_attribute('source', **{'type': 'text', 'value': self.hashlookupresult['source']})
+        if 'KnownMalicious' in self.hashlookupresult:
+            hashlookup_object.add_attribute('KnownMalicious', **{'type': 'text', 'value': self.hashlookupresult['KnownMalicious']})
+        if 'MD5' in self.hashlookupresult:
+            hashlookup_object.add_attribute('MD5', **{'type': 'md5', 'value': self.hashlookupresult['MD5']})
+        # SHA-1 is the default value in hashlookup it must always be present
+        hashlookup_object.add_attribute('SHA-1', **{'type': 'sha1', 'value': self.hashlookupresult['SHA-1']})
+        if 'SHA-256' in self.hashlookupresult:
+            hashlookup_object.add_attribute('SHA-256', **{'type': 'sha256', 'value': self.hashlookupresult['SHA-256']})
+        if 'SSDEEP' in self.hashlookupresult:
+            hashlookup_object.add_attribute('SSDEEP', **{'type': 'ssdeep', 'value': self.hashlookupresult['SSDEEP']})
+        if 'TLSH' in self.hashlookupresult:
+            hashlookup_object.add_attribute('TLSH', **{'type': 'tlsh', 'value': self.hashlookupresult['TLSH']})
+        if 'FileName' in self.hashlookupresult:
+            hashlookup_object.add_attribute('FileName', **{'type': 'filename', 'value': self.hashlookupresult['FileName']})
+        if 'FileSize' in self.hashlookupresult:
+            hashlookup_object.add_attribute('FileSize', **{'type': 'size-in-bytes', 'value': self.hashlookupresult['FileSize']})
+        hashlookup_object.add_reference(self.attribute['uuid'], 'related-to')
+        self.misp_event.add_object(hashlookup_object)
+
+    def __build_references(self):
+        for object_uuid, references in self.references.items():
+            for misp_object in self.misp_event.objects:
+                if misp_object.uuid == object_uuid:
+                    for reference in references:
+                        misp_object.add_reference(**reference)
+                    break
+
+def check_url(url):
+    return "{}/".format(url) if not url.endswith('/') else url
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute.get('type') == 'md5':
+        pass
+    elif attribute.get('type') == 'sha1':
+        pass
+    elif attribute.get('type') == 'sha256':
+        pass
+    else:
+        misperrors['error'] = 'md5 or sha1 or sha256 is missing.'
+        return misperrors
+    api_url = check_url(request['config']['custom_API']) if request['config'].get('custom_API') else hashlookup_url
+    r = requests.get("{}/lookup/{}/{}".format(api_url, attribute.get('type'), attribute['value']))
+    if r.status_code == 200:
+        hashlookupresult = r.json()
+        if not hashlookupresult:
+            misperrors['error'] = 'Empty result'
+            return misperrors
+    elif r.status_code == 404:
+        misperrors['error'] = 'Non existing hash'
+        return misperrors
+    else:
+        misperrors['error'] = 'API not accessible'
+        return misperrors
+    parser = HashlookupParser(attribute, hashlookupresult, api_url)
+    parser.parse_hashlookup_information()
+    result = parser.get_result()
+    return result
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/hibp.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+import requests
+import json
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']}
+moduleinfo = {
+    'version': '0.2',
+    'author': 'Corsin Camichel, Aurélien Schwab',
+    'description': 'Module to access haveibeenpwned.com API.',
+    'module-type': ['hover'],
+    'name': 'Have I Been Pwned Lookup',
+    'logo': 'hibp.png',
+    'requirements': [],
+    'features': 'The module takes an email address as input and queries haveibeenpwned.com API to find additional information about it. This additional information actually tells if any account using the email address has already been compromised in a data breach.',
+    'references': ['https://haveibeenpwned.com/'],
+    'input': 'An email address',
+    'output': 'Additional information about the email address.',
+}
+moduleconfig = ['api_key']
+
+haveibeenpwned_api_url = 'https://haveibeenpwned.com/api/v3/breachedaccount/'
+API_KEY = ""  # details at https://www.troyhunt.com/authentication-and-the-have-i-been-pwned-api/
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    for input_type in mispattributes['input']:
+        if input_type in request:
+            email = request[input_type]
+            break
+    else:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    if request.get('config') is None or request['config'].get('api_key') is None:
+        misperrors['error'] = 'Have I Been Pwned authentication is incomplete (no API key)'
+        return misperrors
+    else:
+        API_KEY = request['config'].get('api_key')
+
+    r = requests.get(haveibeenpwned_api_url + email, headers={'hibp-api-key': API_KEY})
+    if r.status_code == 200:
+        breaches = json.loads(r.text)
+        if breaches:
+            return {'results': [{'types': mispattributes['output'], 'values': breaches}]}
+    elif r.status_code == 404:
+        return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]}
+    else:
+        misperrors['error'] = f'haveibeenpwned.com API not accessible (HTTP {str(r.status_code)})'
+        return misperrors['error']
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/html_to_markdown.py

@@ -0,0 +1,63 @@
+import json
+import requests
+from markdownify import markdownify
+from bs4 import BeautifulSoup
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['url'], 'output': ['text']}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'Sami Mokaddem',
+    'description': 'Expansion module to fetch the html content from an url and convert it into markdown.',
+    'module-type': ['expansion'],
+    'name': 'HTML to Markdown',
+    'logo': '',
+    'requirements': ['The markdownify python library'],
+    'features': 'The module take an URL as input and the HTML content is fetched from it. This content is then converted into markdown that is returned as text.',
+    'references': [],
+    'input': 'URL attribute.',
+    'output': 'Markdown content converted from the HTML fetched from the url.',
+}
+
+
+def fetchHTML(url):
+    r = requests.get(url)
+    return r.text
+
+
+def stripUselessTags(html):
+    soup = BeautifulSoup(html, 'html.parser')
+    toRemove = ['script', 'head', 'header', 'footer', 'meta', 'link']
+    for tag in soup.find_all(toRemove):
+        tag.decompose()
+    return str(soup)
+
+
+def convertHTML(html):
+    toStrip = ['a', 'img']
+    return markdownify(html, heading_style='ATX', strip=toStrip)
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if request.get('url'):
+        url = request['url']
+    else:
+        return False
+    html = fetchHTML(url)
+    html = stripUselessTags(html)
+    markdown = convertHTML(html)
+
+    r = {'results': [{'types': mispattributes['output'],
+                      'values':[str(markdown)]}]}
+    return r
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    return moduleinfo