pymisp 2.5.7.1__py3-none-any.whl → 2.5.8.1__py3-none-any.whl

examples/falsepositive_disabletoids.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-'''
-Koen Van Impe
-
-Disable the to_ids flag of an attribute when there are to many false positives
-Put this script in crontab to run every /15 or /60
-    */5 *    * * *   mispuser   /usr/bin/python3 /home/mispuser/PyMISP/examples/falsepositive_disabletoids.py
-
-Do inline config in "main"
-
-'''
-
-from pymisp import ExpandedPyMISP, MISPEvent
-from keys import misp_url, misp_key, misp_verifycert
-from datetime import datetime
-from datetime import date
-
-import datetime as dt
-import smtplib
-import mimetypes
-from email.mime.multipart import MIMEMultipart
-from email import encoders
-from email.mime.base import MIMEBase
-from email.mime.text import MIMEText
-import argparse
-
-
-def init(url, key, verifycert):
-    '''
-        Template to get MISP module started
-    '''
-    return ExpandedPyMISP(url, key, verifycert, 'json')
-
-
-if __name__ == '__main__':
-
-    minimal_fp = 0
-    threshold_to_ids = .50
-    minimal_date_sighting_date = '1970-01-01 00:00:00'
-
-    smtp_from = 'INSERT_FROM'
-    smtp_to = 'INSERT_TO'
-    smtp_server = 'localhost'
-    report_changes = ''
-    ts_format = '%Y-%m-%d %H:%M:%S'
-
-    parser = argparse.ArgumentParser(description="Disable the to_ids flag of attributes with a certain number of false positives above a threshold.")
-    parser.add_argument('-m', '--mail', action='store_true', help='Mail the report')
-    parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'')
-    parser.add_argument('-b', '--minimal-fp', default=minimal_fp, type=int, help='Minimal number of false positive (default: %(default)s )')
-    parser.add_argument('-t', '--threshold', default=threshold_to_ids, type=float, help='Threshold false positive/true positive rate (default: %(default)s )')
-    parser.add_argument('-d', '--minimal-date-sighting', default=minimal_date_sighting_date, help='Minimal date for sighting (false positive / true positive) (default: %(default)s )')
-
-    args = parser.parse_args()
-    misp = init(misp_url, misp_key, misp_verifycert)
-
-    minimal_fp = int(args.minimal_fp)
-    threshold_to_ids = args.threshold
-    minimal_date_sighting_date = args.minimal_date_sighting
-    minimal_date_sighting = int(dt.datetime.strptime(minimal_date_sighting_date, '%Y-%m-%d %H:%M:%S').strftime("%s"))
-
-    # Fetch all the attributes
-    result = misp.search('attributes', to_ids=1, include_sightings=1)
-
-    if 'Attribute' in result:
-        for attribute in result['Attribute']:
-            true_positive = 0
-            false_positive = 0
-            compute_threshold = 0
-            attribute_id = attribute['id']
-            attribute_value = attribute['value']
-            attribute_uuid = attribute['uuid']
-            event_id = attribute['event_id']
-
-            # Only do something if there is a sighting
-            if 'Sighting' in attribute:
-
-                for sighting in attribute['Sighting']:
-                    if int(sighting['date_sighting']) > minimal_date_sighting:
-                        if int(sighting['type']) == 0:
-                            true_positive = true_positive + 1
-                        elif int(sighting['type']) == 1:
-                            false_positive = false_positive + 1
-
-            if false_positive > minimal_fp:
-                compute_threshold = false_positive / (true_positive + false_positive)
-
-                if compute_threshold >= threshold_to_ids:
-                    # Fetch event title for report text
-                    event_details = misp.get_event(event_id)
-                    event_info = event_details['Event']['info']
-
-                    misp.update_attribute( { 'uuid': attribute_uuid, 'to_ids': 0})
-
-                    report_changes = report_changes + 'Disable to_ids for [%s] (%s) in event [%s] (%s) - FP: %s TP: %s \n' % (attribute_value, attribute_id, event_info, event_id, false_positive, true_positive)
-
-                    # Changing the attribute to_ids flag sets the event to unpublished
-                    misp.publish(event_id)
-
-    # Only send/print the report if it contains content
-    if report_changes:
-        if args.mail:
-            if args.mailoptions:
-                mailoptions = args.mailoptions.split(';')
-                for s in mailoptions:
-                    if s.split('=')[0] == 'smtp_from':
-                        smtp_from = s.split('=')[1]
-                    if s.split('=')[0] == 'smtp_to':
-                        smtp_to = s.split('=')[1]
-                    if s.split('=')[0] == 'smtp_server':
-                        smtp_server = s.split('=')[1]
-
-            now = datetime.now()
-            current_date = now.strftime(ts_format)
-            report_changes_body = 'MISP Disable to_ids flags for %s on %s\n-------------------------------------------------------------------------------\n\n' % (misp_url, current_date)
-            report_changes_body = report_changes_body + 'Minimal number of false positives before considering threshold: %s\n' % (minimal_fp)
-            report_changes_body = report_changes_body + 'Threshold false positives/true positives to disable to_ids flag: %s\n' % (threshold_to_ids)
-            report_changes_body = report_changes_body + 'Minimal date for sighting false positives: %s\n\n' % (minimal_date_sighting_date)
-            report_changes_body = report_changes_body + report_changes
-            report_changes_body = report_changes_body + '\nEvents that have attributes with changed to_ids flag have been republished, without e-mail notification.'
-            report_changes_body = report_changes_body + '\n\nMISP Disable to_ids Finished\n'
-
-            subject = 'Report of disable to_ids flag for false positives sightings of %s' % (current_date)
-            msg = MIMEMultipart()
-            msg['From'] = smtp_from
-            msg['To'] = smtp_to
-            msg['Subject'] = subject
-
-            msg.attach(MIMEText(report_changes_body, 'text'))
-            print(report_changes_body)
-            server = smtplib.SMTP(smtp_server)
-            server.sendmail(smtp_from, smtp_to, msg.as_string())
-
-        else:
-            print(report_changes)

examples/fetch_events_feed.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from keys import misp_url, misp_key, misp_verifycert
-import argparse
-from pymisp import ExpandedPyMISP
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Fetch all events from a feed.')
-    parser.add_argument("-f", "--feed", required=True, help="feed's ID to be fetched.")
-    args = parser.parse_args()
-
-    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
-    misp.fetch_feed(args.feed)

examples/fetch_warninglist_hits.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import PyMISP
-from keys import misp_url, misp_key
-import argparse
-
-
-def init(url, key):
-    return PyMISP(url, key)
-
-
-def loop_attributes(elem):
-    if 'Attribute' in elem.keys():
-        for attribute in elem['Attribute']:
-            if 'warnings' in attribute.keys():
-                for warning in attribute['warnings']:
-                    print("Value {} has a hit in warninglist with name '{}' and id '{}'".format(warning['value'],
-                                                                                                warning[
-                                                                                                    'warninglist_name'],
-                                                                                                warning[
-                                                                                                    'warninglist_id']))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Print all warninglist hits for an event.')
-    parser.add_argument("eventid", type=str, help="The event id of the event to get info of")
-    args = parser.parse_args()
-    misp = init(misp_url, misp_key)
-    evt = misp.search('events', eventid=args.eventid, includeWarninglistHits=1)['response'][0]['Event']
-    if 'warnings' in evt.keys():
-        print('warnings in entire event:')
-        print(str(evt['warnings']) + '\n')
-        print('Warnings at attribute levels:')
-        loop_attributes(evt)
-        if 'Object' in evt.keys():
-            for obj in evt['Object']:
-                loop_attributes(obj)

examples/freetext.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import ExpandedPyMISP
-from keys import misp_url, misp_key, misp_verifycert
-import argparse
-
-from io import open
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description="Update a MISP event.")
-    parser.add_argument("-e", "--event", required=True, help="Event ID to update.")
-    parser.add_argument("-i", "--input", required=True, help="Input file")
-
-    args = parser.parse_args()
-
-    pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
-
-    with open(args.input, 'r') as f:
-        result = pymisp.freetext(args.event, f.read())
-    print(result)

examples/generate_file_objects.py

@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import argparse
-import json
-
-try:
-    from pymisp import pymisp_json_default, AbstractMISP
-    from pymisp.tools import make_binary_objects
-except ImportError:
-    pass
-
-
-def check():
-    missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
-    try:
-        import pymisp  # noqa
-    except ImportError:
-        missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp'
-    try:
-        import pydeep  # noqa
-    except ImportError:
-        missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git'
-    try:
-        import lief  # noqa
-    except ImportError:
-        missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
-    try:
-        import magic  # noqa
-    except ImportError:
-        missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.'
-    return json.dumps(missing_dependencies)
-
-
-def make_objects(path):
-    to_return = {'objects': [], 'references': []}
-    fo, peo, seos = make_binary_objects(path)
-
-    if seos:
-        for s in seos:
-            to_return['objects'].append(s)
-            if s.ObjectReference:
-                to_return['references'] += s.ObjectReference
-
-    if peo:
-        if hasattr(peo, 'certificates') and hasattr(peo, 'signers'):
-            # special authenticode case for PE objects
-            for c in peo.certificates:
-                to_return['objects'].append(c)
-            for s in peo.signers:
-                to_return['objects'].append(s)
-            del peo.certificates
-            del peo.signers
-        del peo.sections
-        to_return['objects'].append(peo)
-        if peo.ObjectReference:
-            to_return['references'] += peo.ObjectReference
-
-    if fo:
-        to_return['objects'].append(fo)
-        if fo.ObjectReference:
-            to_return['references'] += fo.ObjectReference
-    return json.dumps(to_return, default=pymisp_json_default)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument("-p", "--path", help="Path to process.")
-    group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.")
-    args = parser.parse_args()
-    a = AbstractMISP()
-
-    if args.check:
-        print(check())
-    if args.path:
-        obj = make_objects(args.path)
-        print(obj)

examples/generate_meta_feed.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from pymisp.tools import feed_meta_generator
-import argparse
-from pathlib import Path
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Build meta files for feed')
-    parser.add_argument("--feed", required=True, help="Path to directory containing the feed.")
-    args = parser.parse_args()
-
-    feed = Path(args.feed)
-
-    feed_meta_generator(feed)

examples/get.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import ExpandedPyMISP
-from keys import misp_url, misp_key, misp_verifycert
-import argparse
-import os
-
-
-proxies = {
-    'http': 'http://127.0.0.1:8123',
-    'https': 'http://127.0.0.1:8123',
-}
-
-proxies = None
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Get an event from a MISP instance.')
-    parser.add_argument("-e", "--event", required=True, help="Event ID to get.")
-    parser.add_argument("-o", "--output", help="Output file")
-
-    args = parser.parse_args()
-
-    if args.output is not None and os.path.exists(args.output):
-        print('Output file already exists, abort.')
-        exit(0)
-
-    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, proxies=proxies)
-
-    event = misp.get_event(args.event, pythonify=True)
-    if args.output:
-        with open(args.output, 'w') as f:
-            f.write(event.to_json())
-    else:
-        print(event.to_json())

examples/get_csv.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import argparse
-
-from pymisp import ExpandedPyMISP
-from keys import misp_url, misp_key, misp_verifycert
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get MISP stuff as CSV.')
-    parser.add_argument("--controller", default='attributes', help="Attribute to use for the search (events, objects, attributes)")
-    parser.add_argument("-e", "--event_id", help="Event ID to fetch. Without it, it will fetch the whole database.")
-    parser.add_argument("-a", "--attribute", nargs='+', help="Attribute column names")
-    parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names")
-    parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)")
-    parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)")
-    parser.add_argument("-f", "--outfile", help="Output file to write the CSV.")
-
-    args = parser.parse_args()
-    pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True)
-    attr = []
-    if args.attribute:
-        attr += args.attribute
-    if args.object_attribute:
-        attr += args.object_attribute
-    if not attr:
-        attr = None
-    print(args.context)
-    response = pymisp.search(return_format='csv', controller=args.controller, eventid=args.event_id, requested_attributes=attr,
-                             type_attribute=args.misp_types, include_context=args.context)
-
-    if args.outfile:
-        with open(args.outfile, 'w') as f:
-            f.write(response)
-    else:
-        print(response)

examples/get_network_activity.py

@@ -1,187 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-    Python script to extract network activity from MISP database
-
-    Koen Van Impe       20141116
-        netflow         20150804
-    Feed it a list of event_id's (1 id per line) with the option "-f".
-    Use --no-comment to get a flat list of entries without event id and title information
-
-    Usage
-        ./get_network_activity.py --netflow --event 8
-            get netflow filter for event 8
-
-        ./get_network_activity.py -f get_network_activity.event_id --netflow
-            get netflow filter for events in id file
-
-        ./get_network_activity.py -f get_network_activity.event_id
-            get output with comments
-"""
-
-from pymisp import PyMISP
-
-from keys import misp_key
-from keys import misp_url
-from keys import misp_verifycert
-
-source = None
-
-
-def init():
-    """
-    Initialize PyMISP
-    Get configuration settings from config file
-    """
-    global source
-    source = PyMISP(misp_url, misp_key, misp_verifycert, 'json')
-
-
-def get_event(event_id):
-    """
-    Get details of an event and add it to the result arrays
-    :event_id   the id of the event
-    """
-    global network_ip_src, network_ip_dst, network_hostname, network_domain
-    global app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printcomment, app_netflow
-
-    event_id = int(event_id)
-    if event_id > 0:
-        event_json = source.get_event(event_id)
-        event_core = event_json["Event"]
-        # event_threatlevel_id = event_core["threat_level_id"]
-
-        # attribute_count = event_core["attribute_count"]
-        attribute = event_core["Attribute"]
-
-        for attribute in event_core["Attribute"]:
-            if app_ids_only and not attribute["to_ids"]:
-                continue
-
-            value = attribute["value"]
-            title = event_core["info"]
-            if app_netflow:
-                app_printcomment = False
-                if attribute["type"] == "ip-dst" and app_ip_dst:
-                    network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")])
-            else:
-                if attribute["type"] == "ip-src" and app_ip_src:
-                    network_ip_src.append([build_entry(value, event_id, title, "ip-src")])
-                elif attribute["type"] == "ip-dst" and app_ip_dst:
-                    network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")])
-                elif attribute["type"] == "domain" and app_domain:
-                    network_domain.append([build_entry(value, event_id, title, "domain")])
-                elif attribute["type"] == "hostname" and app_hostname:
-                    network_hostname.append([build_entry(value, event_id, title, "hostname")])
-                else:
-                    continue
-    else:
-        print("Not a valid ID")
-        return
-
-
-def build_entry(value, event_id, title, source):
-    """
-    Build the line containing the entry
-
-        :value      the datavalue of the entry
-        :event_id   id of the event
-        :title      name of the event
-        :source     from which set was the entry retrieved
-    """
-    global app_printcomment
-
-    if app_printcomment:
-        if app_printtitle:
-            return "%s # Event: %s / %s (from %s) " % (value, event_id, title, source)
-        else:
-            return "%s # Event: %s (from %s) " % (value, event_id, source)
-    else:
-        return value
-
-
-def print_events():
-    """
-    Print the events from the result arrays
-    """
-    global network_ip_src, network_ip_dst, network_domain, network_hostname
-    global app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printcomment, app_printtitle, app_netflow
-
-    if app_netflow:
-        firsthost = True
-        for ip in network_ip_dst:
-            if firsthost:
-                firsthost = False
-            else:
-                print(" or ")
-            print("host %s" % ip[0])
-    else:
-        if app_ip_src:
-            for ip in network_ip_src:
-                print(ip[0])
-        if app_ip_dst:
-            for ip in network_ip_dst:
-                print(ip[0])
-        if app_domain:
-            for ip in network_domain:
-                print(ip[0])
-        if app_hostname:
-            for ip in network_hostname:
-                print(ip[0])
-
-
-if __name__ == '__main__':
-    import argparse
-
-    network_ip_src = []
-    network_ip_dst = []
-    network_domain = []
-    network_hostname = []
-
-    parser = argparse.ArgumentParser(
-        description='Download network activity information from MISP.')
-    parser.add_argument('-f', '--filename', type=str,
-                        help='File containing a list of event id.')
-    parser.add_argument('--hostname', action='store_true', default=False,
-                        help='Include hostnames.')
-    parser.add_argument('--no-ip-src', action='store_true', default=False,
-                        help='Do not include ip-src.')
-    parser.add_argument('--no-ip-dst', action='store_true', default=False,
-                        help='Do not include ip-dst.')
-    parser.add_argument('--domain', action='store_true', default=False,
-                        help='Include domains.')
-    parser.add_argument('--no-comment', action='store_false', default=True,
-                        help='Do not include comment in the output.')
-    parser.add_argument('--no-ids-only', action='store_true', default=False,
-                        help='Include IDS and non-IDS attribures.')
-    parser.add_argument('--no-titles', action='store_true', default=False,
-                        help='Do not include titles')
-    parser.add_argument('--netflow', action='store_true', default=False,
-                        help='Netflow (nfdump) output')
-    parser.add_argument('--event', type=int, default=0,
-                        help='EventID to parse (not using filename)')
-    args = parser.parse_args()
-
-    init()
-    app_printcomment = args.no_comment
-    app_hostname = args.hostname
-    app_domain = args.domain
-    app_ip_src = not(args.no_ip_src)
-    app_ip_dst = not(args.no_ip_dst)
-    app_ids_only = args.no_ids_only
-    app_printtitle = not(args.no_titles)
-    app_netflow = args.netflow
-    app_event = args.event
-
-    if app_event > 0:
-        get_event(app_event)
-        print_events()
-    elif args.filename is not None:
-        # print "app_printcomment %s app_hostname %s app_domain %s app_ip_src %s app_ip_dst %s app_ids_only %s app_printtitle %s" % (app_printcomment,app_hostname, app_domain, app_ip_src, app_ip_dst, app_ids_only, app_printtitle)
-        with open(args.filename, 'r') as line:
-            for event_id in line:
-                get_event(event_id.strip())
-        print_events()
-    else:
-        print("No filename given, stopping.")

examples/last.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import ExpandedPyMISP
-from keys import misp_url, misp_key, misp_verifycert
-try:
-    from keys import misp_client_cert
-except ImportError:
-    misp_client_cert = ''
-import argparse
-import os
-
-
-# Usage for pipe masters: ./last.py -l 5h | jq .
-# Usage in case of large data set and pivoting page by page: python3 last.py  -l 48h  -m 10 -p 2  | jq .[].Event.info
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Download latest events from a MISP instance.')
-    parser.add_argument("-l", "--last", required=True, help="can be defined in days, hours, minutes (for example 5d or 12h or 30m).")
-    parser.add_argument("-m", "--limit", required=False, default="10", help="Add the limit of records to get (by default, the limit is set to 10)")
-    parser.add_argument("-p", "--page", required=False, default="1", help="Add the page to request to paginate over large dataset (by default page is set to 1)")
-    parser.add_argument("-o", "--output", help="Output file")
-
-    args = parser.parse_args()
-
-    if args.output is not None and os.path.exists(args.output):
-        print('Output file already exists, aborted.')
-        exit(0)
-
-    if misp_client_cert == '':
-        misp_client_cert = None
-    else:
-        misp_client_cert = (misp_client_cert)
-
-    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, cert=misp_client_cert)
-    result = misp.search(publish_timestamp=args.last, limit=args.limit, page=args.page, pythonify=True)
-
-    if not result:
-        print('No results for that time period')
-        exit(0)
-
-    if args.output:
-        with open(args.output, 'w') as f:
-            for r in result:
-                f.write(r.to_json() + '\n')
-    else:
-        for r in result:
-            print(r.to_json())