pymisp 2.5.3__py3-none-any.whl → 2.5.7__py3-none-any.whl

examples/falsepositive_disabletoids.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+'''
+Koen Van Impe
+
+Disable the to_ids flag of an attribute when there are to many false positives
+Put this script in crontab to run every /15 or /60
+    */5 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/falsepositive_disabletoids.py
+
+Do inline config in "main"
+
+'''
+
+from pymisp import ExpandedPyMISP, MISPEvent
+from keys import misp_url, misp_key, misp_verifycert
+from datetime import datetime
+from datetime import date
+
+import datetime as dt
+import smtplib
+import mimetypes
+from email.mime.multipart import MIMEMultipart
+from email import encoders
+from email.mime.base import MIMEBase
+from email.mime.text import MIMEText
+import argparse
+
+
+def init(url, key, verifycert):
+    '''
+        Template to get MISP module started
+    '''
+    return ExpandedPyMISP(url, key, verifycert, 'json')
+
+
+if __name__ == '__main__':
+
+    minimal_fp = 0
+    threshold_to_ids = .50
+    minimal_date_sighting_date = '1970-01-01 00:00:00'
+
+    smtp_from = 'INSERT_FROM'
+    smtp_to = 'INSERT_TO'
+    smtp_server = 'localhost'
+    report_changes = ''
+    ts_format = '%Y-%m-%d %H:%M:%S'
+
+    parser = argparse.ArgumentParser(description="Disable the to_ids flag of attributes with a certain number of false positives above a threshold.")
+    parser.add_argument('-m', '--mail', action='store_true', help='Mail the report')
+    parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'')
+    parser.add_argument('-b', '--minimal-fp', default=minimal_fp, type=int, help='Minimal number of false positive (default: %(default)s )')
+    parser.add_argument('-t', '--threshold', default=threshold_to_ids, type=float, help='Threshold false positive/true positive rate (default: %(default)s )')
+    parser.add_argument('-d', '--minimal-date-sighting', default=minimal_date_sighting_date, help='Minimal date for sighting (false positive / true positive) (default: %(default)s )')
+
+    args = parser.parse_args()
+    misp = init(misp_url, misp_key, misp_verifycert)
+
+    minimal_fp = int(args.minimal_fp)
+    threshold_to_ids = args.threshold
+    minimal_date_sighting_date = args.minimal_date_sighting
+    minimal_date_sighting = int(dt.datetime.strptime(minimal_date_sighting_date, '%Y-%m-%d %H:%M:%S').strftime("%s"))
+
+    # Fetch all the attributes
+    result = misp.search('attributes', to_ids=1, include_sightings=1)
+
+    if 'Attribute' in result:
+        for attribute in result['Attribute']:
+            true_positive = 0
+            false_positive = 0
+            compute_threshold = 0
+            attribute_id = attribute['id']
+            attribute_value = attribute['value']
+            attribute_uuid = attribute['uuid']
+            event_id = attribute['event_id']
+
+            # Only do something if there is a sighting
+            if 'Sighting' in attribute:
+
+                for sighting in attribute['Sighting']:
+                    if int(sighting['date_sighting']) > minimal_date_sighting:
+                        if int(sighting['type']) == 0:
+                            true_positive = true_positive + 1
+                        elif int(sighting['type']) == 1:
+                            false_positive = false_positive + 1
+
+            if false_positive > minimal_fp:
+                compute_threshold = false_positive / (true_positive + false_positive)
+
+                if compute_threshold >= threshold_to_ids:
+                    # Fetch event title for report text
+                    event_details = misp.get_event(event_id)
+                    event_info = event_details['Event']['info']
+
+                    misp.update_attribute( { 'uuid': attribute_uuid, 'to_ids': 0})
+
+                    report_changes = report_changes + 'Disable to_ids for [%s] (%s) in event [%s] (%s) - FP: %s TP: %s \n' % (attribute_value, attribute_id, event_info, event_id, false_positive, true_positive)
+
+                    # Changing the attribute to_ids flag sets the event to unpublished
+                    misp.publish(event_id)
+
+    # Only send/print the report if it contains content
+    if report_changes:
+        if args.mail:
+            if args.mailoptions:
+                mailoptions = args.mailoptions.split(';')
+                for s in mailoptions:
+                    if s.split('=')[0] == 'smtp_from':
+                        smtp_from = s.split('=')[1]
+                    if s.split('=')[0] == 'smtp_to':
+                        smtp_to = s.split('=')[1]
+                    if s.split('=')[0] == 'smtp_server':
+                        smtp_server = s.split('=')[1]
+
+            now = datetime.now()
+            current_date = now.strftime(ts_format)
+            report_changes_body = 'MISP Disable to_ids flags for %s on %s\n-------------------------------------------------------------------------------\n\n' % (misp_url, current_date)
+            report_changes_body = report_changes_body + 'Minimal number of false positives before considering threshold: %s\n' % (minimal_fp)
+            report_changes_body = report_changes_body + 'Threshold false positives/true positives to disable to_ids flag: %s\n' % (threshold_to_ids)
+            report_changes_body = report_changes_body + 'Minimal date for sighting false positives: %s\n\n' % (minimal_date_sighting_date)
+            report_changes_body = report_changes_body + report_changes
+            report_changes_body = report_changes_body + '\nEvents that have attributes with changed to_ids flag have been republished, without e-mail notification.'
+            report_changes_body = report_changes_body + '\n\nMISP Disable to_ids Finished\n'
+
+            subject = 'Report of disable to_ids flag for false positives sightings of %s' % (current_date)
+            msg = MIMEMultipart()
+            msg['From'] = smtp_from
+            msg['To'] = smtp_to
+            msg['Subject'] = subject
+
+            msg.attach(MIMEText(report_changes_body, 'text'))
+            print(report_changes_body)
+            server = smtplib.SMTP(smtp_server)
+            server.sendmail(smtp_from, smtp_to, msg.as_string())
+
+        else:
+            print(report_changes)

examples/fetch_events_feed.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+from pymisp import ExpandedPyMISP
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Fetch all events from a feed.')
+    parser.add_argument("-f", "--feed", required=True, help="feed's ID to be fetched.")
+    args = parser.parse_args()
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+    misp.fetch_feed(args.feed)

examples/fetch_warninglist_hits.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key
+import argparse
+
+
+def init(url, key):
+    return PyMISP(url, key)
+
+
+def loop_attributes(elem):
+    if 'Attribute' in elem.keys():
+        for attribute in elem['Attribute']:
+            if 'warnings' in attribute.keys():
+                for warning in attribute['warnings']:
+                    print("Value {} has a hit in warninglist with name '{}' and id '{}'".format(warning['value'],
+                                                                                                warning[
+                                                                                                    'warninglist_name'],
+                                                                                                warning[
+                                                                                                    'warninglist_id']))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Print all warninglist hits for an event.')
+    parser.add_argument("eventid", type=str, help="The event id of the event to get info of")
+    args = parser.parse_args()
+    misp = init(misp_url, misp_key)
+    evt = misp.search('events', eventid=args.eventid, includeWarninglistHits=1)['response'][0]['Event']
+    if 'warnings' in evt.keys():
+        print('warnings in entire event:')
+        print(str(evt['warnings']) + '\n')
+        print('Warnings at attribute levels:')
+        loop_attributes(evt)
+        if 'Object' in evt.keys():
+            for obj in evt['Object']:
+                loop_attributes(obj)

examples/freetext.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+
+from io import open
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Update a MISP event.")
+    parser.add_argument("-e", "--event", required=True, help="Event ID to update.")
+    parser.add_argument("-i", "--input", required=True, help="Input file")
+
+    args = parser.parse_args()
+
+    pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
+
+    with open(args.input, 'r') as f:
+        result = pymisp.freetext(args.event, f.read())
+    print(result)

examples/generate_file_objects.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import json
+
+try:
+    from pymisp import pymisp_json_default, AbstractMISP
+    from pymisp.tools import make_binary_objects
+except ImportError:
+    pass
+
+
+def check():
+    missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
+    try:
+        import pymisp  # noqa
+    except ImportError:
+        missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp'
+    try:
+        import pydeep  # noqa
+    except ImportError:
+        missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git'
+    try:
+        import lief  # noqa
+    except ImportError:
+        missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
+    try:
+        import magic  # noqa
+    except ImportError:
+        missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.'
+    return json.dumps(missing_dependencies)
+
+
+def make_objects(path):
+    to_return = {'objects': [], 'references': []}
+    fo, peo, seos = make_binary_objects(path)
+
+    if seos:
+        for s in seos:
+            to_return['objects'].append(s)
+            if s.ObjectReference:
+                to_return['references'] += s.ObjectReference
+
+    if peo:
+        if hasattr(peo, 'certificates') and hasattr(peo, 'signers'):
+            # special authenticode case for PE objects
+            for c in peo.certificates:
+                to_return['objects'].append(c)
+            for s in peo.signers:
+                to_return['objects'].append(s)
+            del peo.certificates
+            del peo.signers
+        del peo.sections
+        to_return['objects'].append(peo)
+        if peo.ObjectReference:
+            to_return['references'] += peo.ObjectReference
+
+    if fo:
+        to_return['objects'].append(fo)
+        if fo.ObjectReference:
+            to_return['references'] += fo.ObjectReference
+    return json.dumps(to_return, default=pymisp_json_default)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("-p", "--path", help="Path to process.")
+    group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.")
+    args = parser.parse_args()
+    a = AbstractMISP()
+
+    if args.check:
+        print(check())
+    if args.path:
+        obj = make_objects(args.path)
+        print(obj)

examples/generate_meta_feed.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp.tools import feed_meta_generator
+import argparse
+from pathlib import Path
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Build meta files for feed')
+    parser.add_argument("--feed", required=True, help="Path to directory containing the feed.")
+    args = parser.parse_args()
+
+    feed = Path(args.feed)
+
+    feed_meta_generator(feed)

examples/get.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+import argparse
+import os
+
+
+proxies = {
+    'http': 'http://127.0.0.1:8123',
+    'https': 'http://127.0.0.1:8123',
+}
+
+proxies = None
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
+    parser.add_argument("-e", "--event", required=True, help="Event ID to get.")
+    parser.add_argument("-o", "--output", help="Output file")
+
+    args = parser.parse_args()
+
+    if args.output is not None and os.path.exists(args.output):
+        print('Output file already exists, abort.')
+        exit(0)
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, proxies=proxies)
+
+    event = misp.get_event(args.event, pythonify=True)
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(event.to_json())
+    else:
+        print(event.to_json())

examples/get_csv.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get MISP stuff as CSV.')
+    parser.add_argument("--controller", default='attributes', help="Attribute to use for the search (events, objects, attributes)")
+    parser.add_argument("-e", "--event_id", help="Event ID to fetch. Without it, it will fetch the whole database.")
+    parser.add_argument("-a", "--attribute", nargs='+', help="Attribute column names")
+    parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names")
+    parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)")
+    parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)")
+    parser.add_argument("-f", "--outfile", help="Output file to write the CSV.")
+
+    args = parser.parse_args()
+    pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True)
+    attr = []
+    if args.attribute:
+        attr += args.attribute
+    if args.object_attribute:
+        attr += args.object_attribute
+    if not attr:
+        attr = None
+    print(args.context)
+    response = pymisp.search(return_format='csv', controller=args.controller, eventid=args.event_id, requested_attributes=attr,
+                             type_attribute=args.misp_types, include_context=args.context)
+
+    if args.outfile:
+        with open(args.outfile, 'w') as f:
+            f.write(response)
+    else:
+        print(response)

examples/get_network_activity.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    Python script to extract network activity from MISP database
+
+    Koen Van Impe       20141116
+        netflow         20150804
+    Feed it a list of event_id's (1 id per line) with the option "-f".
+    Use --no-comment to get a flat list of entries without event id and title information
+
+    Usage
+        ./get_network_activity.py --netflow --event 8
+            get netflow filter for event 8
+
+        ./get_network_activity.py -f get_network_activity.event_id --netflow
+            get netflow filter for events in id file
+
+        ./get_network_activity.py -f get_network_activity.event_id
+            get output with comments
+"""
+
+from pymisp import PyMISP
+
+from keys import misp_key
+from keys import misp_url
+from keys import misp_verifycert
+
+source = None
+
+
+def init():
+    """
+    Initialize PyMISP
+    Get configuration settings from config file
+    """
+    global source
+    source = PyMISP(misp_url, misp_key, misp_verifycert, 'json')
+
+
+def get_event(event_id):
+    """
+    Get details of an event and add it to the result arrays
+    :event_id   the id of the event
+    """
+    global network_ip_src, network_ip_dst, network_hostname, network_domain
+    global app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printcomment, app_netflow
+
+    event_id = int(event_id)
+    if event_id > 0:
+        event_json = source.get_event(event_id)
+        event_core = event_json["Event"]
+        # event_threatlevel_id = event_core["threat_level_id"]
+
+        # attribute_count = event_core["attribute_count"]
+        attribute = event_core["Attribute"]
+
+        for attribute in event_core["Attribute"]:
+            if app_ids_only and not attribute["to_ids"]:
+                continue
+
+            value = attribute["value"]
+            title = event_core["info"]
+            if app_netflow:
+                app_printcomment = False
+                if attribute["type"] == "ip-dst" and app_ip_dst:
+                    network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")])
+            else:
+                if attribute["type"] == "ip-src" and app_ip_src:
+                    network_ip_src.append([build_entry(value, event_id, title, "ip-src")])
+                elif attribute["type"] == "ip-dst" and app_ip_dst:
+                    network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")])
+                elif attribute["type"] == "domain" and app_domain:
+                    network_domain.append([build_entry(value, event_id, title, "domain")])
+                elif attribute["type"] == "hostname" and app_hostname:
+                    network_hostname.append([build_entry(value, event_id, title, "hostname")])
+                else:
+                    continue
+    else:
+        print("Not a valid ID")
+        return
+
+
+def build_entry(value, event_id, title, source):
+    """
+    Build the line containing the entry
+
+        :value      the datavalue of the entry
+        :event_id   id of the event
+        :title      name of the event
+        :source     from which set was the entry retrieved
+    """
+    global app_printcomment
+
+    if app_printcomment:
+        if app_printtitle:
+            return "%s # Event: %s / %s (from %s) " % (value, event_id, title, source)
+        else:
+            return "%s # Event: %s (from %s) " % (value, event_id, source)
+    else:
+        return value
+
+
+def print_events():
+    """
+    Print the events from the result arrays
+    """
+    global network_ip_src, network_ip_dst, network_domain, network_hostname
+    global app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printcomment, app_printtitle, app_netflow
+
+    if app_netflow:
+        firsthost = True
+        for ip in network_ip_dst:
+            if firsthost:
+                firsthost = False
+            else:
+                print(" or ")
+            print("host %s" % ip[0])
+    else:
+        if app_ip_src:
+            for ip in network_ip_src:
+                print(ip[0])
+        if app_ip_dst:
+            for ip in network_ip_dst:
+                print(ip[0])
+        if app_domain:
+            for ip in network_domain:
+                print(ip[0])
+        if app_hostname:
+            for ip in network_hostname:
+                print(ip[0])
+
+
+if __name__ == '__main__':
+    import argparse
+
+    network_ip_src = []
+    network_ip_dst = []
+    network_domain = []
+    network_hostname = []
+
+    parser = argparse.ArgumentParser(
+        description='Download network activity information from MISP.')
+    parser.add_argument('-f', '--filename', type=str,
+                        help='File containing a list of event id.')
+    parser.add_argument('--hostname', action='store_true', default=False,
+                        help='Include hostnames.')
+    parser.add_argument('--no-ip-src', action='store_true', default=False,
+                        help='Do not include ip-src.')
+    parser.add_argument('--no-ip-dst', action='store_true', default=False,
+                        help='Do not include ip-dst.')
+    parser.add_argument('--domain', action='store_true', default=False,
+                        help='Include domains.')
+    parser.add_argument('--no-comment', action='store_false', default=True,
+                        help='Do not include comment in the output.')
+    parser.add_argument('--no-ids-only', action='store_true', default=False,
+                        help='Include IDS and non-IDS attribures.')
+    parser.add_argument('--no-titles', action='store_true', default=False,
+                        help='Do not include titles')
+    parser.add_argument('--netflow', action='store_true', default=False,
+                        help='Netflow (nfdump) output')
+    parser.add_argument('--event', type=int, default=0,
+                        help='EventID to parse (not using filename)')
+    args = parser.parse_args()
+
+    init()
+    app_printcomment = args.no_comment
+    app_hostname = args.hostname
+    app_domain = args.domain
+    app_ip_src = not(args.no_ip_src)
+    app_ip_dst = not(args.no_ip_dst)
+    app_ids_only = args.no_ids_only
+    app_printtitle = not(args.no_titles)
+    app_netflow = args.netflow
+    app_event = args.event
+
+    if app_event > 0:
+        get_event(app_event)
+        print_events()
+    elif args.filename is not None:
+        # print "app_printcomment %s app_hostname %s app_domain %s app_ip_src %s app_ip_dst %s app_ids_only %s app_printtitle %s" % (app_printcomment,app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printtitle)
+        with open(args.filename, 'r') as line:
+            for event_id in line:
+                get_event(event_id.strip())
+        print_events()
+    else:
+        print("No filename given, stopping.")

examples/last.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import ExpandedPyMISP
+from keys import misp_url, misp_key, misp_verifycert
+try:
+    from keys import misp_client_cert
+except ImportError:
+    misp_client_cert = ''
+import argparse
+import os
+
+
+# Usage for pipe masters: ./last.py -l 5h | jq .
+# Usage in case of large data set and pivoting page by page: python3 last.py  -l 48h  -m 10 -p 2  | jq .[].Event.info
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Download latest events from a MISP instance.')
+    parser.add_argument("-l", "--last", required=True, help="can be defined in days, hours, minutes (for example 5d or 12h or 30m).")
+    parser.add_argument("-m", "--limit", required=False, default="10", help="Add the limit of records to get (by default, the limit is set to 10)")
+    parser.add_argument("-p", "--page", required=False, default="1", help="Add the page to request to paginate over large dataset (by default page is set to 1)")
+    parser.add_argument("-o", "--output", help="Output file")
+
+    args = parser.parse_args()
+
+    if args.output is not None and os.path.exists(args.output):
+        print('Output file already exists, aborted.')
+        exit(0)
+
+    if misp_client_cert == '':
+        misp_client_cert = None
+    else:
+        misp_client_cert = (misp_client_cert)
+
+    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, cert=misp_client_cert)
+    result = misp.search(publish_timestamp=args.last, limit=args.limit, page=args.page, pythonify=True)
+
+    if not result:
+        print('No results for that time period')
+        exit(0)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            for r in result:
+                f.write(r.to_json() + '\n')
+    else:
+        for r in result:
+            print(r.to_json())