pymisp 2.5.7__py3-none-any.whl → 2.5.8__py3-none-any.whl

examples/vt_to_misp.py

@@ -1,182 +0,0 @@
-''' Convert a VirusTotal report into MISP objects '''
-import argparse
-import json
-import logging
-from datetime import datetime
-from urllib.parse import urlsplit
-
-import pymisp
-from pymisp.tools import VTReportObject
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(module)s.%(funcName)s.%(lineno)d | %(message)s")
-
-
-def build_cli():
-    '''
-    Build the command-line arguments
-    '''
-    desc = "Take an indicator or list of indicators to search VT for and import the results into MISP"
-    post_desc = """
-config.json: Should be a JSON file containing MISP and VirusTotal credentials with the following format:
-{"misp": {"url": "<url_to_misp>", "key": "<misp_api_key>"}, "virustotal": {"key": "<vt_api_key>"}}
-Please note: Only public API features work in the VTReportObject for now. I don't have a quarter million to spare ;)
-
-Example:
-    python vt_to_misp.py -i 719c97a8cd8db282586c1416894dcaf8 -c ./config.json
-    """
-    parser = argparse.ArgumentParser(description=desc, epilog=post_desc, formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument("-e", "--event", help="MISP event id to add to")
-    parser.add_argument("-c", "--config", default="config.json", help="Path to JSON configuration file to read")
-    indicators = parser.add_mutually_exclusive_group(required=True)
-    indicators.add_argument("-i", "--indicator", help="Single indicator to look up")
-    indicators.add_argument("-f", "--file", help="File of indicators to look up - one on each line")
-    indicators.add_argument("-l", "--link", help="Link to a VirusTotal report")
-    return parser.parse_args()
-
-
-def build_config(path=None):
-    '''
-    Read a configuration file path. File is expected to be
-
-    :path: Path to a configuration file
-    '''
-    try:
-        with open(path, "r") as ifile:
-            return json.load(ifile)
-    except OSError:
-        raise OSError("Couldn't find path to configuration file: %s", path)
-    except json.JSONDecodeError:
-        raise IOError("Couldn't parse configuration file. Please make sure it is a proper JSON document")
-
-
-def generate_report(indicator, apikey):
-    '''
-    Build our VirusTotal report object, File object, and AV signature objects
-    and link them appropriately
-
-    :indicator: Indicator hash to search in VT for
-    '''
-    report_objects = []
-    vt_report = VTReportObject(apikey, indicator)
-    report_objects.append(vt_report)
-    raw_report = vt_report._report
-    if vt_report._resource_type == "file":
-        file_object = pymisp.MISPObject(name="file")
-        file_object.add_attribute("md5", value=raw_report["md5"])
-        file_object.add_attribute("sha1", value=raw_report["sha1"])
-        file_object.add_attribute("sha256", value=raw_report["sha256"])
-        vt_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of")
-        report_objects.append(file_object)
-    elif vt_report._resource_type == "url":
-        parsed = urlsplit(indicator)
-        url_object = pymisp.MISPObject(name="url")
-        url_object.add_attribute("url", value=parsed.geturl())
-        url_object.add_attribute("host", value=parsed.hostname)
-        url_object.add_attribute("scheme", value=parsed.scheme)
-        url_object.add_attribute("port", value=parsed.port)
-        vt_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of")
-        report_objects.append(url_object)
-    for antivirus in raw_report["scans"]:
-        if raw_report["scans"][antivirus]["detected"]:
-            av_object = pymisp.MISPObject(name="av-signature")
-            av_object.add_attribute("software", value=antivirus)
-            signature_name = raw_report["scans"][antivirus]["result"]
-            av_object.add_attribute("signature", value=signature_name, disable_correlation=True)
-            vt_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in")
-            report_objects.append(av_object)
-    return report_objects
-
-
-def get_misp_event(event_id=None, info=None):
-    '''
-    Smaller helper function for generating a new MISP event or using a preexisting one
-
-    :event_id: The event id of the MISP event to upload objects to
-
-    :info: The event's title/info
-    '''
-    if event_id:
-        event = misp.get_event(event_id)
-    elif info:
-        event = misp.new_event(info=info)
-    else:
-        event = misp.new_event(info="VirusTotal Report")
-    misp_event = pymisp.MISPEvent()
-    misp_event.load(event)
-    return misp_event
-
-
-def main(misp, config, args):
-    '''
-    Main program logic
-
-    :misp: PyMISP API object for interfacing with MISP
-
-    :config: Configuration dictionary
-
-    :args: Argparse CLI object
-    '''
-    if args.indicator:
-        misp_objects = generate_report(args.indicator, config["virustotal"]["key"])
-        if misp_objects:
-            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(args.indicator))
-            submit_to_misp(misp, misp_event, misp_objects)
-    elif args.file:
-        try:
-            reports = []
-            with open(args.file, "r") as ifile:
-                for indicator in ifile:
-                    try:
-                        misp_objects = generate_report(indicator, config["virustotal"]["key"])
-                        if misp_objects:
-                            reports.append(misp_objects)
-                    except pymisp.exceptions.InvalidMISPObject as err:
-                        logging.error(err)
-            if reports:
-                current_time = datetime.now().strftime("%x %X")
-                misp_event = get_misp_event(args.event, "VirusTotal Reports: {}".format(current_time))
-                for report in reports:
-                    submit_to_misp(misp, misp_event, report)
-        except OSError:
-            logging.error("Couldn't open indicators file at '%s'. Check path", args.file)
-    elif args.link:
-        # https://www.virustotal.com/#/file/<ioc>/detection
-        indicator = args.link.split("/")[5]
-        misp_objects = generate_report(indicator, config["virustotal"]["key"])
-        if misp_objects:
-            misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(indicator))
-            submit_to_misp(misp, misp_event, misp_objects)
-
-
-def submit_to_misp(misp, misp_event, misp_objects):
-    '''
-    Submit a list of MISP objects to a MISP event
-
-    :misp: PyMISP API object for interfacing with MISP
-
-    :misp_event: MISPEvent object
-
-    :misp_objects: List of MISPObject objects. Must be a list
-    '''
-# go through round one and only add MISP objects
-    for misp_object in misp_objects:
-        template_id = misp.get_object_template_id(misp_object.template_uuid)
-        misp.add_object(misp_event.id, template_id, misp_object)
-    # go through round two and add all the object references for each object
-    for misp_object in misp_objects:
-        for reference in misp_object.ObjectReference:
-            misp.add_object_reference(reference)
-
-
-if __name__ == "__main__":
-    try:
-        args = build_cli()
-        config = build_config(args.config)
-        # change the 'ssl' value if you want to verify your MISP's SSL instance
-        misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False)
-        # finally, let's start checking VT and converting the reports
-        main(misp, config, args)
-    except KeyboardInterrupt:
-        print("Bye Felicia")
-    except pymisp.exceptions.InvalidMISPObject as err:
-        logging.error(err)

examples/warninglists.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import PyMISP
-from pymisp.tools import load_warninglists
-import argparse
-from keys import misp_url, misp_key, misp_verifycert
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description='Load the warninglists.')
-    parser.add_argument("-p", "--package", action='store_true', help="from the PyMISPWarninglists package.")
-    parser.add_argument("-r", "--remote", action='store_true', help="from the MISP instance.")
-
-    args = parser.parse_args()
-
-    if args.package:
-        print(load_warninglists.from_package())
-    elif args.remote:
-        pm = PyMISP(misp_url, misp_key, misp_verifycert)
-        print(load_warninglists.from_instance(pm))

examples/yara.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pymisp import PyMISP
-from keys import misp_url, misp_key,misp_verifycert
-import argparse
-import os
-
-
-def init(url, key):
-    return PyMISP(url, key, misp_verifycert, 'json')
-
-
-def get_yara(m, event_id, out=None):
-    ok, rules = m.get_yara(event_id)
-    if not ok:
-        print(rules)
-    elif out is None:
-        print(rules)
-    else:
-        with open(out, 'w') as f:
-            f.write(rules)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get yara rules from an event.')
-    parser.add_argument("-e", "--event", required=True, help="Event ID.")
-    parser.add_argument("-o", "--output", help="Output file")
-
-    args = parser.parse_args()
-
-    if args.output is not None and os.path.exists(args.output):
-        print('Output file already exists, abord.')
-        exit(0)
-
-    misp = init(misp_url, misp_key)
-
-    get_yara(misp, args.event, args.output)

examples/yara_dump.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-'''
-YARA dumper for MISP
-    by Christophe Vandeplas
-'''
-
-import keys
-from pymisp import PyMISP
-import yara
-import re
-
-
-def dirty_cleanup(value):
-    changed = False
-    substitutions = (('”', '"'),
-                     ('“', '"'),
-                     ('″', '"'),
-                     ('`', "'"),
-                     ('\r', ''),
-                     ('Rule ', 'rule ')  # some people write this with the wrong case
-                     # ('$ ', '$'),    # this breaks rules
-                     # ('\t\t', '\n'), # this breaks rules
-                     )
-    for substitution in substitutions:
-        if substitution[0] in value:
-            changed = True
-            value = value.replace(substitution[0], substitution[1])
-    return value, changed
-
-
-misp = PyMISP(keys.misp_url, keys.misp_key, keys.misp_verify, 'json')
-result = misp.search(controller='attributes', type_attribute='yara')
-
-attr_cnt = 0
-attr_cnt_invalid = 0
-attr_cnt_duplicate = 0
-attr_cnt_changed = 0
-yara_rules = []
-yara_rule_names = []
-if result.get('Attribute'):
-    for attribute in result.get('Attribute'):
-        value = attribute['value']
-        event_id = attribute['event_id']
-        attribute_id = attribute['id']
-
-        value = re.sub('^[ \t]*rule ', 'rule misp_e{}_'.format(event_id), value, flags=re.MULTILINE)
-        value, changed = dirty_cleanup(value)
-        if changed:
-            attr_cnt_changed += 1
-        if 'global rule' in value:  # refuse any global rules as they might disable everything
-            continue
-        if 'private rule' in value:  # private rules need some more rewriting
-            priv_rules = re.findall('private rule (\w+)', value, flags=re.MULTILINE)
-            for priv_rule in priv_rules:
-                value = re.sub(priv_rule, 'misp_e{}_{}'.format(event_id, priv_rule), value, flags=re.MULTILINE)
-
-        # compile the yara rule to confirm it's validity
-        # if valid, ignore duplicate rules
-        try:
-            attr_cnt += 1
-            yara.compile(source=value)
-            yara_rules.append(value)
-            # print("Rule e{} a{} OK".format(event_id, attribute_id))
-        except yara.SyntaxError as e:
-            attr_cnt_invalid += 1
-            # print("Rule e{} a{} NOK - {}".format(event_id, attribute_id, e))
-        except yara.Error as e:
-            attr_cnt_invalid += 1
-            print(e)
-            import traceback
-            print(traceback.format_exc())
-
-# remove duplicates - process the full yara rule list and process errors to eliminate duplicate rule names
-all_yara_rules = '\n'.join(yara_rules)
-while True:
-    try:
-        yara.compile(source=all_yara_rules)
-    except yara.SyntaxError as e:
-        if 'duplicated identifier' in e.args[0]:
-            duplicate_rule_names = re.findall('duplicated identifier "(.*)"', e.args[0])
-            for item in duplicate_rule_names:
-                all_yara_rules = all_yara_rules.replace('rule {}'.format(item), 'rule duplicate_{}'.format(item), 1)
-                attr_cnt_duplicate += 1
-            continue
-        else:
-            # This should never happen as all rules were processed before separately. So logically we should only have duplicates.
-            exit("ERROR SyntaxError in rules: {}".format(e.args))
-    break
-
-# save to a file
-fname = 'misp.yara'
-with open(fname, 'w') as f_out:
-    f_out.write(all_yara_rules)
-
-print("")
-print("MISP attributes with YARA rules: total={} valid={} invalid={} duplicate={} changed={}.".format(attr_cnt, attr_cnt - attr_cnt_invalid, attr_cnt_invalid, attr_cnt_duplicate, attr_cnt_changed))
-print("Valid YARA rule file save to file '{}'. Invalid rules/attributes were ignored.".format(fname))

tests/57c4445b-c548-4654-af0b-4be3950d210f.json

@@ -1 +0,0 @@
-{"Event": {"info": "Ransomware - Xorist", "publish_timestamp": "1472548231", "timestamp": "1472541011", "analysis": "2", "Attribute": [{"category": "External analysis", "comment": "Imported via the Freetext Import Tool - Xchecked via VT: b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "uuid": "57c5300c-0560-4146-bfaa-40e802de0b81", "timestamp": "1472540684", "to_ids": false, "value": "https://www.virustotal.com/file/b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68/analysis/1469554268/", "type": "link"}, {"category": "External analysis", "comment": "", "uuid": "57c5310b-dc34-43cb-8b8e-4846950d210f", "timestamp": "1472541011", "to_ids": false, "value": "http://www.xylibox.com/2011/06/have-fun-with-trojan-ransomwin32xorist.html", "type": "link"}, {"category": "Other", "comment": "", "uuid": "57c444c0-8004-48fa-9c33-8aca950d210f", "timestamp": "1472480448", "to_ids": false, "value": "UPX packed", "type": "comment"}, {"category": "Other", "comment": "", "uuid": "57c44648-96f4-45d4-a8eb-453e950d210f", "timestamp": "1472480840", "to_ids": false, "value": "Key: 85350044dF4AC3518D185678A9414A7F,\r\nEncryption rounds:8,\r\nStart offset: 64,\r\nAlgorithm: TEA", "type": "text"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-fb04-457d-87e7-4127950d210f", "timestamp": "1472480394", "to_ids": true, "value": "3Z4wnG9603it23y.exe", "type": "filename"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-454c-4d17-90d1-4d2f950d210f", "timestamp": "1472480395", "to_ids": true, "value": "0749bae92ca336a02c83d126e04ec628", "type": "md5"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-bef0-4ba7-a071-444e950d210f", "timestamp": "1472480394", "to_ids": true, "value": "77b0c41b7d340b8a3d903f21347bbf06aa766b5b", "type": "sha1"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-3fa4-4d65-9ccc-4afa950d210f", "timestamp": "1472480395", "to_ids": true, "value": "b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "type": "sha256"}, {"category": "Persistence mechanism", "comment": "", "uuid": "57c54b0f-27a4-458b-8e63-4455950d210f", "timestamp": "1472547599", "to_ids": true, "value": "Software\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run|%TEMP%\\3Z4wnG9603it23y.exe", "type": "regkey|value"}], "Tag": [{"colour": "#ffffff", "exportable": true, "name": "tlp:white"}, {"colour": "#3d7a00", "exportable": true, "name": "circl:incident-classification=\"malware\""}, {"colour": "#420053", "exportable": true, "name": "ms-caro-malware:malware-type=\"Ransom\""}, {"colour": "#2c4f00", "exportable": true, "name": "malware_classification:malware-category=\"Ransomware\""}], "published": true, "date": "2016-08-29", "Orgc": {"name": "CIRCL", "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f"}, "threat_level_id": "3", "uuid": "57c4445b-c548-4654-af0b-4be3950d210f"}}

tests/csv_testfiles/invalid_fieldnames.csv

@@ -1,11 +0,0 @@
-SHA1,fileName,size
-2a030cc6d84d5785f5e84d0f5888a411d4b06d01,soft.exe,45568
-2abae839362edfe52d9ebe282fb61113d22b331f,sttager.exe,20480
-6995a32e0a4d4f6d0c9b2a00a96d69bff4b83ea7,test443.exe,373911
-87b1f17fbb4a1e8eef4cb31c1c0194b1426c868c,veil.exe,345761
-afc36916a4df934446681ea28bef6add4decb98a,80_http.exe.exe,411850
-f832d94391a8d2d5cf92773e6c912905ec7c40c7,test1.exe,406636
-056823c7891a04b2fec8903eb401ae3291743a54,beca.exe.exe,23808
-b7afa7acf1b7ded2c4e3d0884b5cdaa230d9f82e,shell1.exe,24576
-4b50b6b9157026ab408d966ece02d1cef8045f82,starggge.exe,27136
-6042dfd50d33da40e383baec4a7ef7c75bf17481,8_32.exe,24064

tests/csv_testfiles/valid_fieldnames.csv

@@ -1,4 +0,0 @@
-md5, sha1, sha256
-644087ccca16d2a728ef7685a4106f09, eabd6974ac71efd72d9e0688d5a6131f336d169c, 385e31c97e3a07bbb81513f0cd0979e64e6b014943902efd002f57b21eadd41e
-34187a34d0a3c5d63016c26346371b54, ce8209ff9828aa8cb095bd7d1589fc4d394c298c, 5f815b8a8e77731c9ca2b3a07a27f880ef24d54e458d77bdabbbaf2269fe96c3
-871aa15f4d61c85e1284e1be3f99f705, 236eac0b19f91117b27f1b198a4d8490d99ec2e5, b434bccf0a5ff75b27184e661df751466aef69f35fbd7b8b8692302b8b886262