pymisp 2.5.4__py3-none-any.whl → 2.5.7.1__py3-none-any.whl

examples/sync_sighting.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+'''
+Koen Van Impe
+
+Sync sightings between MISP instances
+
+Put this script in crontab to run every /15 or /60
+    */5 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/sync_sighting.py
+
+Uses a drift file to keep track of latest timestamp synced (config)
+Install on "clients", these push the sightings back to authoritative MISP instance
+
+'''
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key, misp_verifycert
+from keys import misp_authoritive_url, misp_authoritive_key, misp_authoritive_verifycert
+
+import sys
+import time
+
+
+def init(url, key, verifycert):
+    '''
+        Template to get MISP module started
+    '''
+    return PyMISP(url, key, verifycert, 'json')
+
+
+def search_sightings(misp, timestamp, timestamp_now):
+    '''
+        Search all the local sightings
+        Extend the sighting with the attribute UUID
+    '''
+    completed_sightings = []
+
+    try:
+        found_sightings = misp.search_sightings(date_from=timestamp, date_to=timestamp_now)
+    except Exception as e:
+        sys.exit("Unable to search for sightings")
+
+    if found_sightings is not None and 'response' in found_sightings:
+        for s in found_sightings['response']:
+            if 'Sighting' in s:
+                sighting = s['Sighting']
+                if 'attribute_id' in sighting:
+                    attribute_id = sighting['attribute_id']
+
+                    # Query the attribute to get the uuid
+                    # We need this to update the sighting on the other instance
+                    try:
+                        attribute = misp.get_attribute(attribute_id)
+                    except Exception as e:
+                        if module_DEBUG:
+                            print("Unable to fetch attribute UUID for ID %s " % attribute_id)
+                        continue
+
+                    if 'Attribute' in attribute and 'uuid' in attribute['Attribute']:
+                        attribute_uuid = attribute['Attribute']['uuid']
+                        completed_sightings.append({'attribute_uuid': attribute_uuid, 'date_sighting': sighting['date_sighting'], 'source': sighting['source'], 'type': sighting['type'], 'uuid': sighting['uuid']})
+                    else:
+                        if module_DEBUG:
+                            print("No information returned for attribute ID %s " % attribute_id)
+                        continue
+
+    return completed_sightings
+
+
+def sync_sightings(misp, misp_authoritive, found_sightings, verify_before_push, custom_sighting_text):
+    '''
+        Walk through all the sightings
+    '''
+    if found_sightings is not None:
+        for sighting in found_sightings:
+            attribute_uuid = sighting['attribute_uuid']
+            date_sighting = sighting['date_sighting']
+            source = sighting['source']
+            if not source:
+                source = custom_sighting_text
+            type = sighting['type']
+
+            # Fail safe
+            if verify_before_push:
+                if sighting_exists(misp_authoritive, sighting):
+                    continue
+                else:
+                    continue
+            else:
+                push_sighting(misp_authoritive, attribute_uuid, date_sighting, source, type)
+                continue
+        return True
+    return False
+
+
+def push_sighting(misp_authoritive, attribute_uuid, date_sighting, source, type):
+    '''
+        Push sighting to the authoritative server
+    '''
+    if attribute_uuid:
+        try:
+            misp_authoritive.sighting(uuid=attribute_uuid, source=source, type=type, timestamp=date_sighting)
+            if module_DEBUG:
+                print("Pushed sighting for %s on %s" % (attribute_uuid, date_sighting))
+            return True
+        except Exception as e:
+            if module_DEBUG:
+                print("Unable to update attribute %s " % (attribute_uuid))
+            return False
+
+
+def sighting_exists(misp_authoritive, sighting):
+    '''
+        Check if the sighting exists on the authoritative server
+            sightings/restSearch/attribute for uuid is not supported in MISP
+
+            optionally to implement
+    '''
+    return False
+
+
+def set_drift_timestamp(drift_timestamp, drift_timestamp_path):
+    '''
+        Save the timestamp in a (local) file
+    '''
+    try:
+        with open(drift_timestamp_path, 'w+') as f:
+            f.write(str(drift_timestamp))
+        return True
+    except IOError:
+        sys.exit("Unable to write drift_timestamp %s to %s" % (drift_timestamp, drift_timestamp_path))
+        return False
+
+
+def get_drift_timestamp(drift_timestamp_path):
+    '''
+        From when do we start with the sightings?
+    '''
+    try:
+        with open(drift_timestamp_path) as f:
+            drift = f.read()
+            if drift:
+                drift = int(float(drift))
+            else:
+                drift = 0
+    except IOError:
+        drift = 0
+
+    return drift
+
+
+if __name__ == '__main__':
+    misp = init(misp_url, misp_key, misp_verifycert)
+    misp_authoritive = init(misp_authoritive_url, misp_authoritive_key, misp_authoritive_verifycert)
+    drift_timestamp_path = '/home/mispuser/PyMISP/examples/sync_sighting.drift'
+
+    drift_timestamp = get_drift_timestamp(drift_timestamp_path=drift_timestamp_path)
+    timestamp_now = time.time()
+    module_DEBUG = True
+
+    # Get all attribute sightings
+    found_sightings = search_sightings(misp, drift_timestamp, timestamp_now)
+    if found_sightings is not None and len(found_sightings) > 0:
+        if sync_sightings(misp, misp_authoritive, found_sightings, verify_before_push=False, custom_sighting_text="Custom Sighting"):
+            set_drift_timestamp(timestamp_now, drift_timestamp_path)
+            if module_DEBUG:
+                print("Sighting drift file updated to %s " % (timestamp_now))
+        else:
+            sys.exit("Unable to sync sync_sightings")
+    else:
+        sys.exit("No sightings found")

examples/tags.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+import json
+
+
+def get_tags(m):
+    result = m.get_all_tags(True)
+    r = result
+    print(json.dumps(r) + '\n')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get tags from MISP instance.')
+
+    args = parser.parse_args()
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+
+    tags = misp.tags(pythonify=True)
+    for tag in tags:
+        print(tag.to_json())

examples/test_sign.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from pymisp import mispevent
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Sign & verify a MISP event.')
+    parser.add_argument("-i", "--input", required=True, help="Json file")
+    parser.add_argument("-u", "--uid", required=True, help="GPG UID")
+    args = parser.parse_args()
+
+    me = mispevent.MISPEvent()
+    me.load(args.input)
+
+    me.sign(args.uid)
+    me.verify(args.uid)

examples/trustar_misp.py

@@ -0,0 +1,59 @@
+from trustar import TruStar, datetime_to_millis
+from datetime import datetime, timedelta
+from keys import misp_url, misp_key, misp_verifycert
+from pymisp import PyMISP, MISPEvent, MISPOrganisation, MISPObject
+
+# enclave_ids = '7a33144f-aef3-442b-87d4-dbf70d8afdb0'  # RHISAC
+enclave_ids = None
+
+time_interval = {'days': 30, 'hours': 0}
+
+distribution = None  # Optional, defaults to MISP.default_event_distribution in MISP config
+threat_level_id = None  # Optional, defaults to MISP.default_event_threat_level in MISP config
+analysis = None  # Optional, defaults to 0 (initial analysis)
+
+
+
+tru = TruStar()
+
+misp = PyMISP(misp_url, misp_key, misp_verifycert)
+
+now = datetime.now()
+
+# date range for pulling reports is last 4 hours when script is run
+to_time = datetime.now()
+from_time = to_time - timedelta(**time_interval)
+
+# convert to millis since epoch
+to_time = datetime_to_millis(to_time)
+from_time = datetime_to_millis(from_time)
+
+if not enclave_ids:
+    reports = tru.get_reports(from_time=from_time,
+                              to_time=to_time)
+else:
+    reports = tru.get_reports(from_time=from_time,
+                          to_time=to_time,
+                          is_enclave=True,
+                          enclave_ids=enclave_ids)
+
+# loop through each trustar report and create MISP events for each
+for report in reports:
+    # initialize and set MISPEvent()
+    event = MISPEvent()
+    event.info = report.title
+    event.distribution = distribution
+    event.threat_level_id = threat_level_id
+    event.analysis = analysis
+
+    # get tags for report
+    for tag in tru.get_enclave_tags(report.id):
+        event.add_tag(tag.name)
+
+    obj = MISPObject('trustar_report', standalone=False, strict=True)
+    # get indicators for report
+    for indicator in tru.get_indicators_for_report(report.id):
+        obj.add_attribute(indicator.type, indicator.value)
+    event.add_object(obj)
+    # post each event to MISP via API
+    misp.add_event(event)

examples/up.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP, MISPEvent
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Update a MISP event.")
+    parser.add_argument("-e", "--event", required=True, help="Event ID to update.")
+    parser.add_argument("-i", "--input", required=True, help="Input file")
+
+    args = parser.parse_args()
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+
+    me = MISPEvent()
+    me.load_file(args.input)
+
+    result = misp.update_event(me, args.event)

examples/upload.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP, MISPEvent, MISPAttribute
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+from pathlib import Path
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Send malware sample to MISP.')
+    parser.add_argument("-u", "--upload", type=str, required=True, help="File or directory of files to upload.")
+    parser.add_argument("-d", "--distrib", type=int, help="The distribution setting used for the attributes and for the newly created event, if relevant. [0-3].")
+    parser.add_argument("-c", "--comment", type=str, help="Comment for the uploaded file(s).")
+    parser.add_argument('-m', '--is-malware', action='store_true', help='The file(s) to upload are malwares')
+    parser.add_argument('--expand', action='store_true', help='(Only if the file is a malware) Run lief expansion (creates objects)')
+    parser.add_argument("-e", "--event", type=int, default=None, help="Not supplying an event ID will cause MISP to create a single new event for all of the POSTed malware samples.")
+    parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.")
+    args = parser.parse_args()
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+
+    files = []
+    p = Path(args.upload)
+    if p.is_file():
+        files = [p]
+    elif p.is_dir():
+        files = [f for f in p.glob('**/*') if f.is_file()]
+    else:
+        print('invalid upload path (must be file or dir)')
+        exit(0)
+
+    if args.is_malware:
+        arg_type = 'malware-sample'
+    else:
+        arg_type = 'attachment'
+
+    # Create attributes
+    attributes = []
+    for f in files:
+        a = MISPAttribute()
+        a.type = arg_type
+        a.value = f.name
+        a.data = f
+        a.comment = args.comment
+        a.distribution = args.distrib
+        if args.expand and arg_type == 'malware-sample':
+            a.expand = 'binary'
+        attributes.append(a)
+
+    if args.event:
+        for a in attributes:
+            misp.add_attribute(args.event, a)
+    else:
+        m = MISPEvent()
+        m.info = args.info
+        m.distribution = args.distrib
+        m.attributes = attributes
+        if args.expand and arg_type == 'malware-sample':
+            m.run_expansions()
+        misp.add_event(m)

examples/users_list.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.')
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+
+    users_list = misp.users(pythonify=True)
+    print(users_list)

examples/vmray_automation.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Jens Thom (VMRay), Koen Van Impe
+
+VMRay automatic import
+Put this script in crontab to run every /15 or /60
+    */5 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py
+
+Calls "vmray_import" for all events that have an 'incomplete' VMray analysis
+
+Do inline config in "main".
+If your MISP user is not an admin, you cannot use `get_config`,
+use `overwrite_config` instead.
+Example config:
+    config = {
+        "vmray_import_enabled": True,
+        "vmray_import_apikey": vmray_api_key,
+        "vmray_import_url": vmray_server,
+        "vmray_import_disable_tags": False,
+        "vmray_import_disable_misp_objects": False,
+        "vmray_import_ignore_analysis_finished": False,
+        "services_port": 6666,
+        "services_url": "http://localhost",
+        "Artifacts": "1",
+        "VTI": "1",
+        "IOCs": "1",
+        "Analysis Details": "1",
+    }
+"""
+
+import logging
+import urllib
+
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from keys import misp_key, misp_url, misp_verifycert
+from pymisp import ExpandedPyMISP
+
+# Suppress those "Unverified HTTPS request is being made"
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def is_url(url: str) -> bool:
+    try:
+        result = urllib.parse.urlparse(url)
+        return result.scheme and result.netloc
+    except ValueError:
+        return False
+
+
+class VMRayAutomationException(Exception):
+    pass
+
+
+class VMRayAutomation:
+    def __init__(
+        self,
+        misp_url: str,
+        misp_key: str,
+        verify_cert: bool = False,
+        debug: bool = False,
+    ) -> None:
+        # setup logging
+        log_level = logging.DEBUG if debug else logging.INFO
+        log_format = "%(asctime)s - %(name)s - %(levelname)8s - %(message)s"
+
+        logging.basicConfig(level=log_level, format=log_format)
+        logging.getLogger("pymisp").setLevel(log_level)
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+        self.misp_url = misp_url.rstrip("/")
+        self.misp_key = misp_key
+        self.verifycert = verify_cert
+        self.misp = ExpandedPyMISP(misp_url, misp_key, ssl=verify_cert, debug=debug)
+        self.config = {}
+        self.tag_incomplete = 'workflow:state="incomplete"'
+
+    @staticmethod
+    def _setting_enabled(value: bool) -> bool:
+        if not value:
+            raise VMRayAutomationException(
+                "VMRay import is disabled. "
+                "Please enable `vmray_import` in the MISP settings."
+            )
+
+        return True
+
+    @staticmethod
+    def _setting_apikey(value: str) -> str:
+        if not value:
+            raise VMRayAutomationException(
+                "VMRay API key not set. Please set the API key in the MISP settings."
+            )
+
+        return value
+
+    @staticmethod
+    def _setting_url(value: str) -> str:
+        if not value:
+            raise VMRayAutomationException(
+                "VMRay URL not set. Please set the URL in the MISP settings."
+            )
+
+        if not is_url(value):
+            raise VMRayAutomationException("Not a valid URL")
+
+        return value
+
+    @staticmethod
+    def _setting_disabled(value: str) -> bool:
+        return value.lower() in ["no", "false"]
+
+    @staticmethod
+    def _services_port(value: int) -> bool:
+        if value == 0:
+            return 6666
+        return value
+
+    @staticmethod
+    def services_url(value: str) -> bool:
+        if not is_url(value):
+            raise VMRayAutomationException("Services URL is not valid.")
+
+        return value
+
+    @property
+    def vmray_settings(self) -> Dict[str, Any]:
+        return {
+            "vmray_import_enabled": self._setting_enabled,
+            "vmray_import_apikey": self._setting_apikey,
+            "vmray_import_url": self._setting_url,
+            "vmray_import_disable_tags": self._setting_disabled,
+            "vmray_import_disable_misp_objects": self._setting_disabled,
+            "vmray_import_ignore_analysis_finished": self._setting_disabled,
+            "services_port": self._services_port,
+            "services_url": self.services_url,
+        }
+
+    def _get_misp_settings(self) -> List[Dict[str, Any]]:
+        misp_headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "Authorization": self.misp_key,
+        }
+
+        response = requests.get(
+            f"{self.misp_url}/servers/serverSettings.json",
+            verify=self.verifycert,
+            headers=misp_headers,
+        )
+
+        if response.status_code == 200:
+            settings = response.json()
+            if "finalSettings" in settings:
+                return settings["finalSettings"]
+
+        raise VMRayAutomationException("Could not get settings from MISP server.")
+
+    def get_config(self) -> None:
+        self.logger.debug("Loading confing...")
+        # get settings from MISP server
+        settings = self._get_misp_settings()
+        for setting in settings:
+            config_name = setting["setting"].replace("Plugin.Import_", "")
+            if config_name in self.vmray_settings:
+                func = self.vmray_settings[config_name]
+                value = func(setting["value"])
+                self.config[config_name] = value
+
+        # set default `vmray_import` settings
+        self.config.setdefault("VTI", "1")
+        self.config.setdefault("IOCs", "1")
+        self.config.setdefault("Artifacts", "0")
+        self.config.setdefault("Analysis Details", "1")
+
+        self.logger.info("Loading config: Done.")
+
+    def overwrite_config(self, config: Dict[str, Any]) -> None:
+        self.config.update(config)
+
+    def _get_sample_id(self, value: str) -> Optional[int]:
+        vmray_sample_id_text = "VMRay Sample ID: "
+        if not value.startswith(vmray_sample_id_text):
+            self.logger.warning("Invalid Sample ID: %s.", value)
+            return None
+
+        return int(value.replace(vmray_sample_id_text, ""))
+
+    def _call_vmray_import(self, sample_id: int, event_id: str) -> Dict[str, Any]:
+        url = f"{self.config['services_url']}:{self.config['services_port']}/query"
+
+        config = {"Sample ID": sample_id}
+        for key, value in self.config.items():
+            vmray_config_key = key.replace("vmray_import_", "")
+            config[vmray_config_key] = str(value)
+
+        data = {
+            "module": "vmray_import",
+            "event_id": event_id,
+            "config": config,
+            "data": "",
+        }
+
+        self.logger.debug("calling `vmray_import`: url=%s, config=%s", url, config)
+        response = requests.post(url, json=data)
+        if response.status_code != 200:
+            raise VMRayAutomationException(
+                f"MISP modules returned status code `{response.status_code}`"
+            )
+
+        json_response = response.json()
+        if "error" in json_response:
+            error = json_response["error"]
+            raise VMRayAutomationException(f"MISP modules returned error: {error}")
+
+        return json_response
+
+    def _add_event_attributes(self, event_id: int, attributes: Dict[str, Any]) -> None:
+        event = self.misp.get_event(event_id, pythonify=True)
+        for attr in attributes["Attribute"]:
+            event.add_attribute(**attr)
+
+        self.misp.update_event(event)
+
+    def _add_event_objects(self, event_id: int, objects: Dict[str, Any]) -> None:
+        event = self.misp.get_event(event_id, pythonify=True)
+        for obj in objects["Object"]:
+            event.add_object(**obj)
+
+        if "Tag" in objects:
+            for tag in objects["Tag"]:
+                event.add_tag(tag["name"])
+
+        self.misp.update_event(event)
+
+    def _add_misp_event(self, event_id: int, response: Dict[str, Any]) -> None:
+        if self.config["vmray_import_disable_misp_objects"]:
+            self._add_event_attributes(event_id, response["results"])
+        else:
+            self._add_event_objects(event_id, response["results"])
+
+    def import_incomplete_analyses(self) -> None:
+        self.logger.info("Searching for attributes with tag='%s'", self.tag_incomplete)
+        result = self.misp.search("attributes", tags=self.tag_incomplete)
+        attributes = result["Attribute"]
+
+        for attr in attributes:
+            event_id = int(attr["event_id"])
+            self.logger.info("Processing event ID `%d`.", event_id)
+
+            sample_id = self._get_sample_id(attr["value"])
+            if not sample_id:
+                continue
+
+            response = self._call_vmray_import(sample_id, event_id)
+            self._add_misp_event(event_id, response)
+            self.misp.untag(attr["uuid"], self.tag_incomplete)
+
+
+def main():
+    debug = False
+    config = {
+        "Artifacts": "0",
+        "VTI": "1",
+        "IOCs": "1",
+        "Analysis Details": "0",
+        "vmray_import_disable_misp_objects": False,
+    }
+
+    automation = VMRayAutomation(misp_url, misp_key, misp_verifycert, debug)
+    automation.get_config()  # only possible with admin user
+    automation.overwrite_config(config)
+    automation.import_incomplete_analyses()
+
+
+if __name__ == "__main__":
+    main()