pymisp 2.5.4__py3-none-any.whl → 2.5.7__py3-none-any.whl

examples/vt_to_misp.py

@@ -0,0 +1,182 @@
+''' Convert a VirusTotal report into MISP objects '''
+import argparse
+import json
+import logging
+from datetime import datetime
+from urllib.parse import urlsplit
+
+import pymisp
+from pymisp.tools import VTReportObject
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(module)s.%(funcName)s.%(lineno)d | %(message)s")
+
+
+def build_cli():
+    '''
+    Build the command-line arguments
+    '''
+    desc = "Take an indicator or list of indicators to search VT for and import the results into MISP"
+    post_desc = """
+config.json: Should be a JSON file containing MISP and VirusTotal credentials with the following format:
+{"misp": {"url": "<url_to_misp>", "key": "<misp_api_key>"}, "virustotal": {"key": "<vt_api_key>"}}
+Please note: Only public API features work in the VTReportObject for now. I don't have a quarter million to spare ;)
+
+Example:
+    python vt_to_misp.py -i 719c97a8cd8db282586c1416894dcaf8 -c ./config.json
+    """
+    parser = argparse.ArgumentParser(description=desc, epilog=post_desc, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("-e", "--event", help="MISP event id to add to")
+    parser.add_argument("-c", "--config", default="config.json", help="Path to JSON configuration file to read")
+    indicators = parser.add_mutually_exclusive_group(required=True)
+    indicators.add_argument("-i", "--indicator", help="Single indicator to look up")
+    indicators.add_argument("-f", "--file", help="File of indicators to look up - one on each line")
+    indicators.add_argument("-l", "--link", help="Link to a VirusTotal report")
+    return parser.parse_args()
+
+
+def build_config(path=None):
+    '''
+    Read a configuration file path. File is expected to be
+
+    :path: Path to a configuration file
+    '''
+    try:
+        with open(path, "r") as ifile:
+            return json.load(ifile)
+    except OSError:
+        raise OSError("Couldn't find path to configuration file: %s", path)
+    except json.JSONDecodeError:
+        raise IOError("Couldn't parse configuration file. Please make sure it is a proper JSON document")
+
+
+def generate_report(indicator, apikey):
+    '''
+    Build our VirusTotal report object, File object, and AV signature objects
+    and link them appropriately
+
+    :indicator: Indicator hash to search in VT for
+    '''
+    report_objects = []
+    vt_report = VTReportObject(apikey, indicator)
+    report_objects.append(vt_report)
+    raw_report = vt_report._report
+    if vt_report._resource_type == "file":
+        file_object = pymisp.MISPObject(name="file")
+        file_object.add_attribute("md5", value=raw_report["md5"])
+        file_object.add_attribute("sha1", value=raw_report["sha1"])
+        file_object.add_attribute("sha256", value=raw_report["sha256"])
+        vt_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of")
+        report_objects.append(file_object)
+    elif vt_report._resource_type == "url":
+        parsed = urlsplit(indicator)
+        url_object = pymisp.MISPObject(name="url")
+        url_object.add_attribute("url", value=parsed.geturl())
+        url_object.add_attribute("host", value=parsed.hostname)
+        url_object.add_attribute("scheme", value=parsed.scheme)
+        url_object.add_attribute("port", value=parsed.port)
+        vt_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of")
+        report_objects.append(url_object)
+    for antivirus in raw_report["scans"]:
+        if raw_report["scans"][antivirus]["detected"]:
+            av_object = pymisp.MISPObject(name="av-signature")
+            av_object.add_attribute("software", value=antivirus)
+            signature_name = raw_report["scans"][antivirus]["result"]
+            av_object.add_attribute("signature", value=signature_name, disable_correlation=True)
+            vt_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in")
+            report_objects.append(av_object)
+    return report_objects
+
+
+def get_misp_event(event_id=None, info=None):
+    '''
+    Smaller helper function for generating a new MISP event or using a preexisting one
+
+    :event_id: The event id of the MISP event to upload objects to
+
+    :info: The event's title/info
+    '''
+    if event_id:
+        event = misp.get_event(event_id)
+    elif info:
+        event = misp.new_event(info=info)
+    else:
+        event = misp.new_event(info="VirusTotal Report")
+    misp_event = pymisp.MISPEvent()
+    misp_event.load(event)
+    return misp_event
+
+
+def main(misp, config, args):
+    '''
+    Main program logic
+
+    :misp: PyMISP API object for interfacing with MISP
+
+    :config: Configuration dictionary
+
+    :args: Argparse CLI object
+    '''
+    if args.indicator:
+        misp_objects = generate_report(args.indicator, config["virustotal"]["key"])
+        if misp_objects:
+            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(args.indicator))
+            submit_to_misp(misp, misp_event, misp_objects)
+    elif args.file:
+        try:
+            reports = []
+            with open(args.file, "r") as ifile:
+                for indicator in ifile:
+                    try:
+                        misp_objects = generate_report(indicator, config["virustotal"]["key"])
+                        if misp_objects:
+                            reports.append(misp_objects)
+                    except pymisp.exceptions.InvalidMISPObject as err:
+                        logging.error(err)
+            if reports:
+                current_time = datetime.now().strftime("%x %X")
+                misp_event = get_misp_event(args.event, "VirusTotal Reports: {}".format(current_time))
+                for report in reports:
+                    submit_to_misp(misp, misp_event, report)
+        except OSError:
+            logging.error("Couldn't open indicators file at '%s'. Check path", args.file)
+    elif args.link:
+        # https://www.virustotal.com/#/file/<ioc>/detection
+        indicator = args.link.split("/")[5]
+        misp_objects = generate_report(indicator, config["virustotal"]["key"])
+        if misp_objects:
+            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(indicator))
+            submit_to_misp(misp, misp_event, misp_objects)
+
+
+def submit_to_misp(misp, misp_event, misp_objects):
+    '''
+    Submit a list of MISP objects to a MISP event
+
+    :misp: PyMISP API object for interfacing with MISP
+
+    :misp_event: MISPEvent object
+
+    :misp_objects: List of MISPObject objects. Must be a list
+    '''
+# go through round one and only add MISP objects
+    for misp_object in misp_objects:
+        template_id = misp.get_object_template_id(misp_object.template_uuid)
+        misp.add_object(misp_event.id, template_id, misp_object)
+    # go through round two and add all the object references for each object
+    for misp_object in misp_objects:
+        for reference in misp_object.ObjectReference:
+            misp.add_object_reference(reference)
+
+
+if __name__ == "__main__":
+    try:
+        args = build_cli()
+        config = build_config(args.config)
+        # change the 'ssl' value if you want to verify your MISP's SSL instance
+        misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False)
+        # finally, let's start checking VT and converting the reports
+        main(misp, config, args)
+    except KeyboardInterrupt:
+        print("Bye Felicia")
+    except pymisp.exceptions.InvalidMISPObject as err:
+        logging.error(err)

examples/warninglists.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from pymisp.tools import load_warninglists
+import argparse
+from keys import misp_url, misp_key, misp_verifycert
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Load the warninglists.')
+    parser.add_argument("-p", "--package", action='store_true', help="from the PyMISPWarninglists package.")
+    parser.add_argument("-r", "--remote", action='store_true', help="from the MISP instance.")
+
+    args = parser.parse_args()
+
+    if args.package:
+        print(load_warninglists.from_package())
+    elif args.remote:
+        pm = PyMISP(misp_url, misp_key, misp_verifycert)
+        print(load_warninglists.from_instance(pm))

examples/yara.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key,misp_verifycert
+import argparse
+import os
+
+
+def init(url, key):
+    return PyMISP(url, key, misp_verifycert, 'json')
+
+
+def get_yara(m, event_id, out=None):
+    ok, rules = m.get_yara(event_id)
+    if not ok:
+        print(rules)
+    elif out is None:
+        print(rules)
+    else:
+        with open(out, 'w') as f:
+            f.write(rules)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get yara rules from an event.')
+    parser.add_argument("-e", "--event", required=True, help="Event ID.")
+    parser.add_argument("-o", "--output", help="Output file")
+
+    args = parser.parse_args()
+
+    if args.output is not None and os.path.exists(args.output):
+        print('Output file already exists, abord.')
+        exit(0)
+
+    misp = init(misp_url, misp_key)
+
+    get_yara(misp, args.event, args.output)

examples/yara_dump.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+'''
+YARA dumper for MISP
+    by Christophe Vandeplas
+'''
+
+import keys
+from pymisp import PyMISP
+import yara
+import re
+
+
+def dirty_cleanup(value):
+    changed = False
+    substitutions = (('”', '"'),
+                     ('“', '"'),
+                     ('″', '"'),
+                     ('`', "'"),
+                     ('\r', ''),
+                     ('Rule ', 'rule ')  # some people write this with the wrong case
+                     # ('$ ', '$'),    # this breaks rules
+                     # ('\t\t', '\n'), # this breaks rules
+                     )
+    for substitution in substitutions:
+        if substitution[0] in value:
+            changed = True
+            value = value.replace(substitution[0], substitution[1])
+    return value, changed
+
+
+misp = PyMISP(keys.misp_url, keys.misp_key, keys.misp_verify, 'json')
+result = misp.search(controller='attributes', type_attribute='yara')
+
+attr_cnt = 0
+attr_cnt_invalid = 0
+attr_cnt_duplicate = 0
+attr_cnt_changed = 0
+yara_rules = []
+yara_rule_names = []
+if result.get('Attribute'):
+    for attribute in result.get('Attribute'):
+        value = attribute['value']
+        event_id = attribute['event_id']
+        attribute_id = attribute['id']
+
+        value = re.sub('^[ \t]*rule ', 'rule misp_e{}_'.format(event_id), value, flags=re.MULTILINE)
+        value, changed = dirty_cleanup(value)
+        if changed:
+            attr_cnt_changed += 1
+        if 'global rule' in value:  # refuse any global rules as they might disable everything
+            continue
+        if 'private rule' in value:  # private rules need some more rewriting
+            priv_rules = re.findall('private rule (\w+)', value, flags=re.MULTILINE)
+            for priv_rule in priv_rules:
+                value = re.sub(priv_rule, 'misp_e{}_{}'.format(event_id, priv_rule), value, flags=re.MULTILINE)
+
+        # compile the yara rule to confirm it's validity
+        # if valid, ignore duplicate rules
+        try:
+            attr_cnt += 1
+            yara.compile(source=value)
+            yara_rules.append(value)
+            # print("Rule e{} a{} OK".format(event_id, attribute_id))
+        except yara.SyntaxError as e:
+            attr_cnt_invalid += 1
+            # print("Rule e{} a{} NOK - {}".format(event_id, attribute_id, e))
+        except yara.Error as e:
+            attr_cnt_invalid += 1
+            print(e)
+            import traceback
+            print(traceback.format_exc())
+
+# remove duplicates - process the full yara rule list and process errors to eliminate duplicate rule names
+all_yara_rules = '\n'.join(yara_rules)
+while True:
+    try:
+        yara.compile(source=all_yara_rules)
+    except yara.SyntaxError as e:
+        if 'duplicated identifier' in e.args[0]:
+            duplicate_rule_names = re.findall('duplicated identifier "(.*)"', e.args[0])
+            for item in duplicate_rule_names:
+                all_yara_rules = all_yara_rules.replace('rule {}'.format(item), 'rule duplicate_{}'.format(item), 1)
+                attr_cnt_duplicate += 1
+            continue
+        else:
+            # This should never happen as all rules were processed before separately. So logically we should only have duplicates.
+            exit("ERROR SyntaxError in rules: {}".format(e.args))
+    break
+
+# save to a file
+fname = 'misp.yara'
+with open(fname, 'w') as f_out:
+    f_out.write(all_yara_rules)
+
+print("")
+print("MISP attributes with YARA rules: total={} valid={} invalid={} duplicate={} changed={}.".format(attr_cnt, attr_cnt - attr_cnt_invalid, attr_cnt_invalid, attr_cnt_duplicate, attr_cnt_changed))
+print("Valid YARA rule file save to file '{}'. Invalid rules/attributes were ignored.".format(fname))

pymisp/api.py

@@ -32,7 +32,7 @@ from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObje
     MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity, MISPUserSetting, \
     MISPInbox, MISPEventBlocklist, MISPOrganisationBlocklist, MISPEventReport, \
     MISPGalaxyCluster, MISPGalaxyClusterRelation, MISPCorrelationExclusion, MISPDecayingModel, \
-    MISPNote, MISPOpinion, MISPRelationship, AnalystDataBehaviorMixin
+    MISPNote, MISPOpinion, MISPRelationship, MISPAnalystData
 from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types
 
 
@@ -499,6 +499,20 @@ class PyMISP:
         response = self._prepare_request('POST', f'events/contact/{event_id}', data=to_post)
         return self._check_json_response(response)
 
+    def enrich_event(self, event: MISPEvent | int | str | UUID, enrich_with: str | list[str]) -> dict[str, Any]:
+        """Enrich an event with data from one or more module.
+
+        :param event: event to enrich
+        :param enrich_with: module name or list of module names to use for enrichment
+        """
+        event_id = get_uuid_or_id_from_abstract_misp(event)
+        if isinstance(enrich_with, str):
+            enrich_with = [enrich_with]
+
+        to_post = {module_name: True for module_name in enrich_with}
+        response = self._prepare_request('POST', f'/events/enrichEvent/{event_id}', data=to_post)
+        return self._check_json_response(response)
+
     # ## END Event ###
 
     # ## BEGIN Event Report ###
@@ -621,14 +635,14 @@ class PyMISP:
     # ## END Galaxy Cluster ###
 
     # ## BEGIN Analyst Data ###a
-    def get_analyst_data(self, analyst_data: AnalystDataBehaviorMixin | int | str | UUID,
+    def get_analyst_data(self, analyst_data: MISPAnalystData | int | str | UUID,
                          pythonify: bool = False) -> dict[str, Any] | MISPNote | MISPOpinion | MISPRelationship:
         """Get an analyst data from a MISP instance
 
         :param analyst_data: analyst data to get
         :param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
         """
-        if isinstance(analyst_data, AnalystDataBehaviorMixin):
+        if isinstance(analyst_data, MISPAnalystData):
             analyst_data_type = analyst_data.analyst_data_object_type
         else:
             analyst_data_type = 'all'
@@ -666,7 +680,7 @@ class PyMISP:
         :param analyst_data_id: analyst data ID to update
         :param pythonify: Returns a PyMISP Object instead of the plain json output
         """
-        if isinstance(analyst_data, AnalystDataBehaviorMixin):
+        if isinstance(analyst_data, MISPAnalystData):
             analyst_data_type = analyst_data.analyst_data_object_type
         else:
             analyst_data_type = 'all'
@@ -685,7 +699,7 @@ class PyMISP:
 
         :param analyst_data: analyst data to delete
         """
-        if isinstance(analyst_data, AnalystDataBehaviorMixin):
+        if isinstance(analyst_data, MISPAnalystData):
             analyst_data_type = analyst_data.analyst_data_object_type
         else:
             analyst_data_type = 'all'
@@ -1102,6 +1116,20 @@ class PyMISP:
         a.from_dict(**response)
         return a
 
+    def enrich_attribute(self, attribute: MISPAttribute | int | str | UUID, enrich_with: str | list[str]) -> dict[str, Any]:
+        """Enrich an attribute with data from one or more module.
+
+        :param attribute: attribute to enrich
+        :param enrich_with: module name or list of module names to use for enrichment
+        """
+        attribute_id = get_uuid_or_id_from_abstract_misp(attribute)
+        if isinstance(enrich_with, str):
+            enrich_with = [enrich_with]
+
+        to_post = {module_name: True for module_name in enrich_with}
+        response = self._prepare_request('POST', f'/attributes/enrich/{attribute_id}', data=to_post)
+        return self._check_json_response(response)
+
     # ## END Attribute ###
 
     # ## BEGIN Attribute Proposal ###

pymisp/data/misp-objects/objects/instagram-account/definition.json

@@ -0,0 +1,66 @@
+{
+  "attributes": {
+    "account-id": {
+      "description": "Account id.",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "account-name": {
+      "description": "Account name.",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "archive": {
+      "description": "Archive of the account (Internet Archive, Archive.is, etc).",
+      "disable_correlation": true,
+      "misp-attribute": "link",
+      "multiple": true,
+      "ui-priority": 1
+    },
+    "attachment": {
+      "description": "A screen capture or exported list of contacts etc.",
+      "misp-attribute": "attachment",
+      "multiple": true,
+      "ui-priority": 1
+    },
+    "description": {
+      "description": "A description of the user.",
+      "misp-attribute": "text",
+      "ui-priority": 1
+    },
+    "is-verified": {
+      "description": "If the user is verified.",
+      "misp-attribute": "boolean",
+      "multiple": false,
+      "ui-priority": 1
+    },
+    "link": {
+      "description": "Original link to the page (supposed harmless).",
+      "misp-attribute": "link",
+      "ui-priority": 1
+    },
+    "url": {
+      "description": "Original URL location of the page (potentially malicious).",
+      "misp-attribute": "url",
+      "ui-priority": 1
+    },
+    "user-avatar": {
+      "description": "A user profile picture or avatar.",
+      "misp-attribute": "attachment",
+      "multiple": true,
+      "ui-priority": 1
+    }
+  },
+  "description": "Instagram account.",
+  "meta-category": "misc",
+  "name": "instagram-account",
+  "requiredOneOf": [
+    "account-name",
+    "account-id",
+    "description",
+    "archive",
+    "link"
+  ],
+  "uuid": "656ced16-23f3-4322-925d-c9c961684999",
+  "version": 1
+}

pymisp/data/misp-objects/objects/lnk/definition.json

@@ -116,6 +116,12 @@
       "misp-attribute": "text",
       "ui-priority": 0
     },
+    "lnk-mft-object": {
+      "description": "LinkTargetIDList MFT entry. Name:MFTID|SeqID|BTimestamp",
+      "misp-attribute": "text",
+      "multiple": true,
+      "ui-priority": 0
+    },
     "lnk-modification-time": {
       "categories": [
         "Other"
@@ -125,6 +131,11 @@
       "misp-attribute": "datetime",
       "ui-priority": 0
     },
+    "lnk-propertystore-sid": {
+      "description": "SID reference in ExtraData.PropertyStore",
+      "misp-attribute": "text",
+      "ui-priority": 0
+    },
     "lnk-relative-path": {
       "description": "Relative path",
       "disable_correlation": true,
@@ -250,6 +261,7 @@
       "description": "Free text value to attach to the file",
       "disable_correlation": true,
       "misp-attribute": "text",
+      "multiple": true,
       "recommended": false,
       "ui-priority": 1
     },
@@ -275,5 +287,5 @@
     "sha512/256"
   ],
   "uuid": "ad13533e-1853-4da0-a111-33a7ce7e6c09",
-  "version": 1
+  "version": 2
 }

pymisp/data/misp-objects/objects/rmm/definition.json

@@ -0,0 +1,88 @@
+{
+  "attributes": {
+    "api-key": {
+      "description": "Authentication or API key used by the RMM agent",
+      "misp-attribute": "text",
+      "multiple": true,
+      "ui-priority": 0
+    },
+    "comment": {
+      "description": "A description of the RMM.",
+      "misp-attribute": "comment",
+      "ui-priority": 0
+    },
+    "guid": {
+      "description": "GUID or reference of the RMM agent or deployment ID.",
+      "misp-attribute": "text",
+      "multiple": true,
+      "ui-priority": 0
+    },
+    "hostname": {
+      "description": "hostname of the control server for the RMM agent.",
+      "misp-attribute": "hostname",
+      "ui-priority": 0
+    },
+    "name": {
+      "description": "Name of the RMM agent.",
+      "misp-attribute": "text",
+      "ui-priority": 0
+    },
+    "rmm-type": {
+      "description": "Type of the RMM agent described.",
+      "disable_correlation": true,
+      "misp-attribute": "text",
+      "sane_default": [
+        "AnyDesk",
+        "Atera",
+        "ASG",
+        "BeAnywhere",
+        "Domotz",
+        "DWservice",
+        "Fixme.it",
+        "Fleetdeck.io",
+        "GetScreen",
+        "Itarian",
+        "Level.io",
+        "Logmein",
+        "ManageEngine",
+        "MeshCentral",
+        "N-Able",
+        "Pulseway",
+        "RattyRat",
+        "Rport",
+        "Rsocx",
+        "RustDesk",
+        "RustScan",
+        "ScreenConnect",
+        "Splashtop",
+        "SSH",
+        "Tactical RMM",
+        "Teamviewer",
+        "TightVNC",
+        "TrendMicro",
+        "Sorillus",
+        "Xeox",
+        "ZeroTier"
+      ],
+      "ui-priority": 0
+    },
+    "url": {
+      "description": "url of the control server for the RMM agent.",
+      "misp-attribute": "url",
+      "ui-priority": 0
+    }
+  },
+  "description": "An object describing a RMM agent.",
+  "meta-category": "misc",
+  "name": "rmm",
+  "requiredOneOf": [
+    "comment",
+    "rmm-type",
+    "guid",
+    "url",
+    "hostname",
+    "api-key"
+  ],
+  "uuid": "2cdcfb2a-dd74-4d88-ae02-6f381b312666",
+  "version": 4
+}

pymisp/data/misp-objects/objects/target-system/definition.json

@@ -28,12 +28,12 @@
       "ui-priority": 1
     }
   },
-  "description": "Description about an targeted system, this could potentially be a compromissed internal system",
+  "description": "Description about an targeted system, this could potentially be a compromised internal system",
   "meta-category": "internal",
   "name": "target-system",
   "requiredOneOf": [
     "targeted_machine"
   ],
   "uuid": "3110944f-eca0-4c94-9d61-a84d022228a4",
-  "version": 1
+  "version": 2
 }

pymisp/data/misp-objects/schema_objects.json

@@ -68,9 +68,9 @@
             "dkim",
             "dkim-signature",
             "dns-soa-email",
+            "dom-hash",
             "domain",
             "domain|ip",
-            "dom-hash",
             "email",
             "email-attachment",
             "email-body",

pymisp/mispevent.py

@@ -2532,6 +2532,14 @@ class MISPAnalystData(AbstractMISP):
     def analyst_data_object_type(self) -> str:
         return self._analyst_data_object_type
 
+    @property
+    def notes(self) -> list[MISPNote]:
+        return self.Note
+
+    @property
+    def opinions(self) -> list[MISPOpinion]:
+        return self.Opinion
+
     @property
     def org(self) -> MISPOrganisation:
         return self.Org