pymisp 2.5.3__py3-none-any.whl → 2.5.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pymisp might be problematic. Click here for more details.
- CHANGELOG.txt +5380 -0
- examples/__init__.py +0 -0
- examples/add_attributes_from_csv.py +74 -0
- examples/add_email_object.py +29 -0
- examples/add_fail2ban_object.py +86 -0
- examples/add_feed.py +25 -0
- examples/add_file_object.py +47 -0
- examples/add_filetype_object_from_csv.py +53 -0
- examples/add_generic_object.py +26 -0
- examples/add_github_user.py +65 -0
- examples/add_gitlab_user.py +56 -0
- examples/add_named_attribute.py +25 -0
- examples/add_organisations.py +57 -0
- examples/add_ssh_authorized_keys.py +29 -0
- examples/add_user.py +22 -0
- examples/add_vehicle_object.py +22 -0
- examples/addtag2.py +45 -0
- examples/asciidoc_generator.py +114 -0
- examples/cache_all.py +10 -0
- examples/copyTagsFromAttributesToEvent.py +68 -0
- examples/copy_list.py +93 -0
- examples/create_events.py +26 -0
- examples/cytomic_orion.py +549 -0
- examples/del.py +22 -0
- examples/delete_user.py +16 -0
- examples/edit_organisation.py +20 -0
- examples/edit_user.py +20 -0
- examples/falsepositive_disabletoids.py +136 -0
- examples/fetch_events_feed.py +15 -0
- examples/fetch_warninglist_hits.py +38 -0
- examples/freetext.py +22 -0
- examples/generate_file_objects.py +78 -0
- examples/generate_meta_feed.py +15 -0
- examples/get.py +37 -0
- examples/get_csv.py +37 -0
- examples/get_network_activity.py +187 -0
- examples/last.py +48 -0
- examples/load_csv.py +94 -0
- examples/lookup.py +28 -0
- examples/misp2cef.py +71 -0
- examples/misp2clamav.py +52 -0
- examples/openioc_to_misp.py +27 -0
- examples/proofpoint_tap.py +203 -0
- examples/proofpoint_vap.py +65 -0
- examples/search.py +48 -0
- examples/search_attributes_yara.py +40 -0
- examples/search_sighting.py +42 -0
- examples/server_sync_check_conn.py +32 -0
- examples/sharing_groups.py +15 -0
- examples/show_sightings.py +168 -0
- examples/stats_report.py +405 -0
- examples/sync_sighting.py +171 -0
- examples/tags.py +25 -0
- examples/test_sign.py +19 -0
- examples/trustar_misp.py +59 -0
- examples/up.py +21 -0
- examples/upload.py +60 -0
- examples/users_list.py +15 -0
- examples/vmray_automation.py +281 -0
- examples/vt_to_misp.py +182 -0
- examples/warninglists.py +22 -0
- examples/yara.py +38 -0
- examples/yara_dump.py +98 -0
- pymisp/api.py +33 -5
- pymisp/data/misp-objects/objects/android-app/definition.json +8 -2
- pymisp/data/misp-objects/objects/instagram-account/definition.json +66 -0
- pymisp/data/misp-objects/objects/lnk/definition.json +13 -1
- pymisp/data/misp-objects/objects/rmm/definition.json +88 -0
- pymisp/data/misp-objects/objects/spambee-report/definition.json +54 -0
- pymisp/data/misp-objects/objects/target-system/definition.json +2 -2
- pymisp/data/misp-objects/objects/vulnerability/definition.json +5 -4
- pymisp/data/misp-objects/relationships/definition.json +17 -1
- pymisp/data/misp-objects/schema_objects.json +1 -1
- pymisp/mispevent.py +95 -23
- {pymisp-2.5.3.dist-info → pymisp-2.5.7.dist-info}/METADATA +23 -28
- {pymisp-2.5.3.dist-info → pymisp-2.5.7.dist-info}/RECORD +144 -30
- {pymisp-2.5.3.dist-info → pymisp-2.5.7.dist-info}/WHEEL +1 -1
- tests/57c4445b-c548-4654-af0b-4be3950d210f.json +1 -0
- tests/__init__.py +0 -0
- tests/csv_testfiles/invalid_fieldnames.csv +11 -0
- tests/csv_testfiles/valid_fieldnames.csv +4 -0
- tests/email_testfiles/mail_1.eml.zip +0 -0
- tests/email_testfiles/mail_1.msg +0 -0
- tests/email_testfiles/mail_1_bom.eml +858 -0
- tests/email_testfiles/mail_1_headers_only.eml +28 -0
- tests/email_testfiles/mail_2.eml +32 -0
- tests/email_testfiles/mail_3.eml +170 -0
- tests/email_testfiles/mail_3.msg +0 -0
- tests/email_testfiles/mail_4.msg +0 -0
- tests/email_testfiles/mail_5.msg +0 -0
- tests/email_testfiles/mail_multiple_to.eml +15 -0
- tests/email_testfiles/source +1 -0
- tests/git-vuln-finder-quagga.json +1493 -0
- tests/misp_event.json +76 -0
- tests/mispevent_testfiles/attribute.json +21 -0
- tests/mispevent_testfiles/attribute_del.json +23 -0
- tests/mispevent_testfiles/def_param.json +53 -0
- tests/mispevent_testfiles/event.json +8 -0
- tests/mispevent_testfiles/event_obj_attr_tag.json +57 -0
- tests/mispevent_testfiles/event_obj_def_param.json +62 -0
- tests/mispevent_testfiles/event_obj_tag.json +29 -0
- tests/mispevent_testfiles/event_tags.json +18 -0
- tests/mispevent_testfiles/existing_event.json +4599 -0
- tests/mispevent_testfiles/existing_event_edited.json +4601 -0
- tests/mispevent_testfiles/galaxy.json +25 -0
- tests/mispevent_testfiles/malware.json +19 -0
- tests/mispevent_testfiles/malware_exist.json +163 -0
- tests/mispevent_testfiles/misp_custom_obj.json +38 -0
- tests/mispevent_testfiles/overwrite_file/definition.json +457 -0
- tests/mispevent_testfiles/proposals.json +35 -0
- tests/mispevent_testfiles/shadow.json +148 -0
- tests/mispevent_testfiles/sighting.json +5 -0
- tests/mispevent_testfiles/simple.json +2 -0
- tests/mispevent_testfiles/test_object_template/definition.json +29 -0
- tests/new_misp_event.json +34 -0
- tests/reportlab_testfiles/HTML_event.json +1 -0
- tests/reportlab_testfiles/galaxy_1.json +1250 -0
- tests/reportlab_testfiles/image_event.json +2490 -0
- tests/reportlab_testfiles/japanese_test.json +156 -0
- tests/reportlab_testfiles/japanese_test_heavy.json +318 -0
- tests/reportlab_testfiles/long_event.json +3730 -0
- tests/reportlab_testfiles/mainly_objects_1.json +1092 -0
- tests/reportlab_testfiles/mainly_objects_2.json +977 -0
- tests/reportlab_testfiles/sighting_1.json +305 -0
- tests/reportlab_testfiles/sighting_2.json +221 -0
- tests/reportlab_testfiles/to_delete1.json +804 -0
- tests/reportlab_testfiles/to_delete2.json +1 -0
- tests/reportlab_testfiles/to_delete3.json +1 -0
- tests/reportlab_testfiles/very_long_event.json +1006 -0
- tests/reportlab_testoutputs/to_delete1.json.pdf +391 -0
- tests/reportlab_testoutputs/to_delete2.json.pdf +506 -0
- tests/reportlab_testoutputs/to_delete3.json.pdf +277 -0
- tests/search_index_result.json +69 -0
- tests/sharing_groups.json +98 -0
- tests/stix1.xml-utf8 +110 -0
- tests/stix2.json +1 -0
- tests/test_analyst_data.py +123 -0
- tests/test_emailobject.py +157 -0
- tests/test_fileobject.py +20 -0
- tests/test_mispevent.py +473 -0
- tests/test_reportlab.py +431 -0
- tests/testlive_comprehensive.py +3734 -0
- tests/testlive_sync.py +474 -0
- pymisp/data/misp-objects/.git +0 -1
- pymisp/data/misp-objects/.gitchangelog.rc +0 -289
- pymisp/data/misp-objects/.github/workflows/codeql.yml +0 -41
- pymisp/data/misp-objects/.github/workflows/nosetests.yml +0 -39
- pymisp/data/misp-objects/.travis.yml +0 -16
- pymisp/data/misp-objects/LICENSE-software-only.md +0 -661
- pymisp/data/misp-objects/LICENSE.md +0 -36
- pymisp/data/misp-objects/README.md +0 -566
- pymisp/data/misp-objects/docs/time-related-objects.ods +0 -0
- pymisp/data/misp-objects/docs/time-related-objects.pdf +0 -0
- pymisp/data/misp-objects/jq_all_the_things.sh +0 -29
- pymisp/data/misp-objects/tools/adoc_objects.py +0 -145
- pymisp/data/misp-objects/tools/alfred_links_to_relarelationships.py +0 -48
- pymisp/data/misp-objects/tools/list_of_objects.py +0 -50
- pymisp/data/misp-objects/tools/updated.sh +0 -6
- pymisp/data/misp-objects/tools/validate_opposites.sh +0 -17
- pymisp/data/misp-objects/unique_uuid.py +0 -16
- pymisp/data/misp-objects/validate_all.sh +0 -38
- {pymisp-2.5.3.dist-info → pymisp-2.5.7.dist-info}/LICENSE +0 -0
|
@@ -0,0 +1,549 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
'''
|
|
5
|
+
Koen Van Impe
|
|
6
|
+
|
|
7
|
+
Cytomic Automation
|
|
8
|
+
Put this script in crontab to run every /15 or /60
|
|
9
|
+
*/15 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
Fetches the configuration set in the Cytomic Orion enrichment module
|
|
13
|
+
- events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion
|
|
14
|
+
- upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
|
|
15
|
+
- delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
|
|
16
|
+
|
|
17
|
+
'''
|
|
18
|
+
|
|
19
|
+
from pymisp import ExpandedPyMISP
|
|
20
|
+
from keys import misp_url, misp_key, misp_verifycert
|
|
21
|
+
import argparse
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import sys
|
|
25
|
+
import requests
|
|
26
|
+
import json
|
|
27
|
+
import urllib3
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password):
|
|
31
|
+
'''
|
|
32
|
+
Get oAuth2 token
|
|
33
|
+
Configuration settings are fetched first from the MISP module configu
|
|
34
|
+
'''
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
if scope and grant_type and username and password:
|
|
38
|
+
data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password}
|
|
39
|
+
|
|
40
|
+
if token_url and clientid and clientsecret:
|
|
41
|
+
access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret))
|
|
42
|
+
tokens = json.loads(access_token_response.text)
|
|
43
|
+
if 'access_token' in tokens:
|
|
44
|
+
access_token = tokens['access_token']
|
|
45
|
+
return access_token
|
|
46
|
+
else:
|
|
47
|
+
sys.exit('No token received')
|
|
48
|
+
else:
|
|
49
|
+
sys.exit('No token_url, clientid or clientsecret supplied')
|
|
50
|
+
else:
|
|
51
|
+
sys.exit('No scope, grant_type, username or password supplied')
|
|
52
|
+
except Exception:
|
|
53
|
+
sys.exit('Unable to connect to token_url')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def get_config(url, key, misp_verifycert):
|
|
57
|
+
'''
|
|
58
|
+
Get the module config and the settings needed to access the API
|
|
59
|
+
Also contains the settings to do the query
|
|
60
|
+
'''
|
|
61
|
+
try:
|
|
62
|
+
misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
|
|
63
|
+
req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
|
|
64
|
+
if req.status_code == 200:
|
|
65
|
+
req_json = req.json()
|
|
66
|
+
if 'finalSettings' in req_json:
|
|
67
|
+
finalSettings = req_json['finalSettings']
|
|
68
|
+
|
|
69
|
+
clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = ''
|
|
70
|
+
module_enabled = False
|
|
71
|
+
scope = 'orion.api'
|
|
72
|
+
grant_type = 'password'
|
|
73
|
+
limit_upload_events = 50
|
|
74
|
+
limit_upload_attributes = 50
|
|
75
|
+
ttlDays = "1"
|
|
76
|
+
last_attributes = '5d'
|
|
77
|
+
post_threat_level_id = 2
|
|
78
|
+
for el in finalSettings:
|
|
79
|
+
# Is the module enabled?
|
|
80
|
+
if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled':
|
|
81
|
+
module_enabled = el['value']
|
|
82
|
+
if module_enabled is False:
|
|
83
|
+
break
|
|
84
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid':
|
|
85
|
+
clientid = el['value']
|
|
86
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret':
|
|
87
|
+
clientsecret = el['value']
|
|
88
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username':
|
|
89
|
+
username = el['value']
|
|
90
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password':
|
|
91
|
+
password = el['value']
|
|
92
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url':
|
|
93
|
+
api_url = el['value'].replace('\\/', '/')
|
|
94
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url':
|
|
95
|
+
token_url = el['value'].replace('\\/', '/')
|
|
96
|
+
elif el['setting'] == 'MISP.baseurl':
|
|
97
|
+
misp_baseurl = el['value']
|
|
98
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id':
|
|
99
|
+
if el['value']:
|
|
100
|
+
try:
|
|
101
|
+
post_threat_level_id = int(el['value'])
|
|
102
|
+
except:
|
|
103
|
+
continue
|
|
104
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays':
|
|
105
|
+
if el['value']:
|
|
106
|
+
try:
|
|
107
|
+
ttlDays = "{last_days}".format(last_days=int(el['value']))
|
|
108
|
+
except:
|
|
109
|
+
continue
|
|
110
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe':
|
|
111
|
+
if el['value']:
|
|
112
|
+
try:
|
|
113
|
+
last_attributes = "{last_days}d".format(last_days=int(el['value']))
|
|
114
|
+
except:
|
|
115
|
+
continue
|
|
116
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag':
|
|
117
|
+
upload_tag = el['value']
|
|
118
|
+
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag':
|
|
119
|
+
delete_tag = el['value']
|
|
120
|
+
elif el['setting'] == 'Plugin.Enrichment_limit_upload_events':
|
|
121
|
+
if el['value']:
|
|
122
|
+
try:
|
|
123
|
+
limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value']))
|
|
124
|
+
except:
|
|
125
|
+
continue
|
|
126
|
+
elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes':
|
|
127
|
+
if el['value']:
|
|
128
|
+
try:
|
|
129
|
+
limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value']))
|
|
130
|
+
except:
|
|
131
|
+
continue
|
|
132
|
+
else:
|
|
133
|
+
sys.exit('Did not receive a 200 code from MISP')
|
|
134
|
+
|
|
135
|
+
if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type:
|
|
136
|
+
|
|
137
|
+
return {'cytomic_policy': 'Detect',
|
|
138
|
+
'upload_timeframe': last_attributes,
|
|
139
|
+
'upload_tag': upload_tag,
|
|
140
|
+
'delete_tag': delete_tag,
|
|
141
|
+
'upload_ttlDays': ttlDays,
|
|
142
|
+
'post_threat_level_id': post_threat_level_id,
|
|
143
|
+
'clientid': clientid,
|
|
144
|
+
'clientsecret': clientsecret,
|
|
145
|
+
'scope': scope,
|
|
146
|
+
'username': username,
|
|
147
|
+
'password': password,
|
|
148
|
+
'grant_type': grant_type,
|
|
149
|
+
'api_url': api_url,
|
|
150
|
+
'token_url': token_url,
|
|
151
|
+
'misp_baseurl': misp_baseurl,
|
|
152
|
+
'limit_upload_events': limit_upload_events,
|
|
153
|
+
'limit_upload_attributes': limit_upload_attributes}
|
|
154
|
+
else:
|
|
155
|
+
sys.exit('Did not receive all the necessary configuration information from MISP')
|
|
156
|
+
|
|
157
|
+
except Exception as e:
|
|
158
|
+
sys.exit('Unable to get module config from MISP')
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class cytomicobject:
|
|
162
|
+
misp = None
|
|
163
|
+
lst_evtid = None
|
|
164
|
+
lst_attuuid = None
|
|
165
|
+
lst_attuuid_error = None
|
|
166
|
+
endpoint_ioc = None
|
|
167
|
+
api_call_headers = None
|
|
168
|
+
post_data = None
|
|
169
|
+
args = None
|
|
170
|
+
tag = None
|
|
171
|
+
limit_events = None
|
|
172
|
+
limit_attributes = None
|
|
173
|
+
atttype_misp = None
|
|
174
|
+
atttype_cytomic = None
|
|
175
|
+
attlabel_cytomic = None
|
|
176
|
+
att_types = {
|
|
177
|
+
"ip-dst": {"ip": "ipioc"},
|
|
178
|
+
"ip-src": {"ip": "ipioc"},
|
|
179
|
+
"url": {"url": "urlioc"},
|
|
180
|
+
"md5": {"hash": "filehashioc"},
|
|
181
|
+
"domain": {"domain": "domainioc"},
|
|
182
|
+
"hostname": {"domain": "domainioc"},
|
|
183
|
+
"domain|ip": {"domain": "domainioc"},
|
|
184
|
+
"hostname|port": {"domain": "domainioc"}
|
|
185
|
+
}
|
|
186
|
+
debug = True
|
|
187
|
+
error = False
|
|
188
|
+
res = False
|
|
189
|
+
res_msg = None
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def collect_events_ids(cytomicobj, moduleconfig):
|
|
193
|
+
# Get events that contain Cytomic tag.
|
|
194
|
+
try:
|
|
195
|
+
evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True)
|
|
196
|
+
cytomicobj.lst_evtid = ['x', 'y']
|
|
197
|
+
for evt in evt_result:
|
|
198
|
+
evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True)
|
|
199
|
+
if len(evt.tags) > 0:
|
|
200
|
+
for tg in evt.tags:
|
|
201
|
+
if tg.name == cytomicobj.tag:
|
|
202
|
+
if not cytomicobj.lst_evtid:
|
|
203
|
+
cytomicobj.lst_evtid = str(evt['id'])
|
|
204
|
+
else:
|
|
205
|
+
if not evt['id'] in cytomicobj.lst_evtid:
|
|
206
|
+
cytomicobj.lst_evtid.append(str(evt['id']))
|
|
207
|
+
break
|
|
208
|
+
cytomicobj.lst_evtid.remove('x')
|
|
209
|
+
cytomicobj.lst_evtid.remove('y')
|
|
210
|
+
except Exception:
|
|
211
|
+
cytomicobj.error = True
|
|
212
|
+
if cytomicobj.debug:
|
|
213
|
+
sys.exit('Unable to collect events ids')
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def find_eventid(cytomicobj, evtid):
|
|
217
|
+
# Get events that contain Cytomic tag.
|
|
218
|
+
try:
|
|
219
|
+
cytomicobj.res = False
|
|
220
|
+
for id in cytomicobj.lst_evtid:
|
|
221
|
+
if id == evtid:
|
|
222
|
+
cytomicobj.res = True
|
|
223
|
+
break
|
|
224
|
+
except Exception:
|
|
225
|
+
cytomicobj.error = True
|
|
226
|
+
if cytomicobj.debug:
|
|
227
|
+
sys.exit('Unable to collect events ids')
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def print_result_events(cytomicobj):
|
|
231
|
+
try:
|
|
232
|
+
if cytomicobj.res_msg is not None:
|
|
233
|
+
for key, msg in cytomicobj.res_msg.items():
|
|
234
|
+
if msg is not None:
|
|
235
|
+
print(key, msg)
|
|
236
|
+
except Exception:
|
|
237
|
+
cytomicobj.error = True
|
|
238
|
+
if cytomicobj.debug:
|
|
239
|
+
sys.exit('Unable to print result')
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def set_postdata(cytomicobj, moduleconfig, attribute):
|
|
243
|
+
# Set JSON to send to the API.
|
|
244
|
+
try:
|
|
245
|
+
|
|
246
|
+
if cytomicobj.args.upload or cytomicobj.args.events:
|
|
247
|
+
event = attribute['Event']
|
|
248
|
+
event_title = event['info']
|
|
249
|
+
event_id = event['id']
|
|
250
|
+
threat_level_id = int(event['threat_level_id'])
|
|
251
|
+
if moduleconfig['post_threat_level_id'] <= threat_level_id:
|
|
252
|
+
|
|
253
|
+
if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
|
|
254
|
+
post_value = attribute['value'].split('|')[0]
|
|
255
|
+
else:
|
|
256
|
+
post_value = attribute['value']
|
|
257
|
+
|
|
258
|
+
if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
|
|
259
|
+
pass
|
|
260
|
+
else:
|
|
261
|
+
if cytomicobj.post_data is None:
|
|
262
|
+
cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}]
|
|
263
|
+
else:
|
|
264
|
+
if post_value not in str(cytomicobj.post_data):
|
|
265
|
+
cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()})
|
|
266
|
+
else:
|
|
267
|
+
if cytomicobject.debug:
|
|
268
|
+
print('Event %s skipped because of lower threat level' % event_id)
|
|
269
|
+
else:
|
|
270
|
+
event = attribute['Event']
|
|
271
|
+
threat_level_id = int(event['threat_level_id'])
|
|
272
|
+
if moduleconfig['post_threat_level_id'] <= threat_level_id:
|
|
273
|
+
if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
|
|
274
|
+
post_value = attribute['value'].split('|')[0]
|
|
275
|
+
else:
|
|
276
|
+
post_value = attribute['value']
|
|
277
|
+
|
|
278
|
+
if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
|
|
279
|
+
pass
|
|
280
|
+
else:
|
|
281
|
+
if cytomicobj.post_data is None:
|
|
282
|
+
cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}]
|
|
283
|
+
else:
|
|
284
|
+
cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value})
|
|
285
|
+
else:
|
|
286
|
+
if cytomicobject.debug:
|
|
287
|
+
print('Event %s skipped because of lower threat level' % event_id)
|
|
288
|
+
except Exception:
|
|
289
|
+
cytomicobj.error = True
|
|
290
|
+
if cytomicobj.debug:
|
|
291
|
+
sys.exit('Unable to process post-data')
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def send_postdata(cytomicobj, evtid=None):
|
|
295
|
+
# Batch post to upload event attributes.
|
|
296
|
+
try:
|
|
297
|
+
if cytomicobj.post_data is not None:
|
|
298
|
+
if cytomicobj.debug:
|
|
299
|
+
print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data))
|
|
300
|
+
result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False)
|
|
301
|
+
json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text)
|
|
302
|
+
print(result_post_endpoint_ioc)
|
|
303
|
+
if 'true' not in (result_post_endpoint_ioc.text):
|
|
304
|
+
cytomicobj.error = True
|
|
305
|
+
if evtid is not None:
|
|
306
|
+
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
|
307
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
|
308
|
+
else:
|
|
309
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
|
310
|
+
if cytomicobj.debug:
|
|
311
|
+
print('RESULT: {}'.format(json_result_post_endpoint_ioc))
|
|
312
|
+
else:
|
|
313
|
+
if evtid is None:
|
|
314
|
+
cytomicobj.error = True
|
|
315
|
+
except Exception:
|
|
316
|
+
cytomicobj.error = True
|
|
317
|
+
if cytomicobj.debug:
|
|
318
|
+
sys.exit('Unable to post attributes')
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def process_attributes(cytomicobj, moduleconfig, evtid=None):
|
|
322
|
+
# Get attributes to process.
|
|
323
|
+
try:
|
|
324
|
+
for misptype, cytomictypes in cytomicobject.att_types.items():
|
|
325
|
+
cytomicobj.atttype_misp = misptype
|
|
326
|
+
for cytomiclabel, cytomictype in cytomictypes.items():
|
|
327
|
+
cytomicobj.attlabel_cytomic = cytomiclabel
|
|
328
|
+
cytomicobj.atttype_cytomic = cytomictype
|
|
329
|
+
cytomicobj.post_data = None
|
|
330
|
+
icont = 0
|
|
331
|
+
if cytomicobj.args.upload or cytomicobj.args.events:
|
|
332
|
+
cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays'])
|
|
333
|
+
else:
|
|
334
|
+
cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic
|
|
335
|
+
|
|
336
|
+
# Get attributes to upload/delete and prepare JSON
|
|
337
|
+
# If evtid is set; we're called from --events
|
|
338
|
+
if cytomicobject.debug:
|
|
339
|
+
print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp)
|
|
340
|
+
|
|
341
|
+
if evtid is None:
|
|
342
|
+
cytomicobj.error = False
|
|
343
|
+
attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
|
|
344
|
+
else:
|
|
345
|
+
if cytomicobj.error:
|
|
346
|
+
break
|
|
347
|
+
# We don't search with tags; we have an event for which we want to upload all events
|
|
348
|
+
attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
|
|
349
|
+
|
|
350
|
+
cytomicobj.lst_attuuid = ['x', 'y']
|
|
351
|
+
|
|
352
|
+
if len(attr_result['Attribute']) > 0:
|
|
353
|
+
for attribute in attr_result['Attribute']:
|
|
354
|
+
if evtid is not None:
|
|
355
|
+
if cytomicobj.error:
|
|
356
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
|
357
|
+
break
|
|
358
|
+
if icont >= cytomicobj.limit_attributes:
|
|
359
|
+
if not cytomicobj.error and cytomicobj.post_data is not None:
|
|
360
|
+
# Send data to Cytomic
|
|
361
|
+
send_postdata(cytomicobj, evtid)
|
|
362
|
+
if not cytomicobj.error:
|
|
363
|
+
if 'Event: ' + str(evtid) in cytomicobj.res_msg:
|
|
364
|
+
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
|
365
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
|
366
|
+
else:
|
|
367
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
|
368
|
+
else:
|
|
369
|
+
if cytomicobject.debug:
|
|
370
|
+
print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
|
|
371
|
+
|
|
372
|
+
cytomicobj.post_data = None
|
|
373
|
+
if cytomicobj.error:
|
|
374
|
+
if evtid is not None:
|
|
375
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
|
376
|
+
break
|
|
377
|
+
icont = 0
|
|
378
|
+
|
|
379
|
+
if evtid is None:
|
|
380
|
+
event = attribute['Event']
|
|
381
|
+
event_id = event['id']
|
|
382
|
+
find_eventid(cytomicobj, str(event_id))
|
|
383
|
+
if not cytomicobj.res:
|
|
384
|
+
if not cytomicobj.lst_attuuid:
|
|
385
|
+
cytomicobj.lst_attuuid = attribute['uuid']
|
|
386
|
+
else:
|
|
387
|
+
if not attribute['uuid'] in cytomicobj.lst_attuuid:
|
|
388
|
+
cytomicobj.lst_attuuid.append(attribute['uuid'])
|
|
389
|
+
icont += 1
|
|
390
|
+
# Prepare data to send
|
|
391
|
+
set_postdata(cytomicobj, moduleconfig, attribute)
|
|
392
|
+
else:
|
|
393
|
+
icont += 1
|
|
394
|
+
# Prepare data to send
|
|
395
|
+
set_postdata(cytomicobj, moduleconfig, attribute)
|
|
396
|
+
|
|
397
|
+
if not cytomicobj.error:
|
|
398
|
+
# Send data to Cytomic
|
|
399
|
+
send_postdata(cytomicobj, evtid)
|
|
400
|
+
|
|
401
|
+
if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0:
|
|
402
|
+
# Data sent; process response
|
|
403
|
+
if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg:
|
|
404
|
+
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
|
405
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
|
406
|
+
else:
|
|
407
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
|
408
|
+
else:
|
|
409
|
+
if cytomicobject.debug:
|
|
410
|
+
print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
|
|
411
|
+
|
|
412
|
+
if not cytomicobj.error:
|
|
413
|
+
cytomicobj.lst_attuuid.remove('x')
|
|
414
|
+
cytomicobj.lst_attuuid.remove('y')
|
|
415
|
+
# Untag attributes
|
|
416
|
+
untag_attributes(cytomicobj)
|
|
417
|
+
except Exception:
|
|
418
|
+
cytomicobj.error = True
|
|
419
|
+
if cytomicobj.debug:
|
|
420
|
+
sys.exit('Unable to get attributes')
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def untag_event(evtid):
|
|
424
|
+
# Remove tag of the event being processed.
|
|
425
|
+
try:
|
|
426
|
+
cytomicobj.records = 0
|
|
427
|
+
evt = cytomicobj.misp.get_event(event=evtid, pythonify=True)
|
|
428
|
+
if len(evt.tags) > 0:
|
|
429
|
+
for tg in evt.tags:
|
|
430
|
+
if tg.name == cytomicobj.tag:
|
|
431
|
+
cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag)
|
|
432
|
+
cytomicobj.records += 1
|
|
433
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)'
|
|
434
|
+
break
|
|
435
|
+
except Exception:
|
|
436
|
+
cytomicobj.error = True
|
|
437
|
+
if cytomicobj.debug:
|
|
438
|
+
sys.exit('Unable to untag events')
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def process_events(cytomicobj, moduleconfig):
|
|
442
|
+
# Get events that contain Cytomic tag.
|
|
443
|
+
try:
|
|
444
|
+
collect_events_ids(cytomicobj, moduleconfig)
|
|
445
|
+
total_attributes_sent = 0
|
|
446
|
+
for evtid in cytomicobj.lst_evtid:
|
|
447
|
+
cytomicobj.error = False
|
|
448
|
+
if cytomicobj.res_msg is None:
|
|
449
|
+
cytomicobj.res_msg = {'Event: ' + str(evtid): None}
|
|
450
|
+
else:
|
|
451
|
+
cytomicobj.res_msg['Event: ' + str(evtid)] = None
|
|
452
|
+
if cytomicobject.debug:
|
|
453
|
+
print('Event id: ' + str(evtid))
|
|
454
|
+
|
|
455
|
+
# get attributes of each known type of the event / prepare data to send / send data to Cytomic
|
|
456
|
+
process_attributes(cytomicobj, moduleconfig, evtid)
|
|
457
|
+
if not cytomicobj.error:
|
|
458
|
+
untag_event(evtid)
|
|
459
|
+
except Exception:
|
|
460
|
+
cytomicobj.error = True
|
|
461
|
+
if cytomicobj.debug:
|
|
462
|
+
sys.exit('Unable to process events ids')
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def untag_attributes(cytomicobj):
|
|
466
|
+
# Remove tag of attributes sent.
|
|
467
|
+
try:
|
|
468
|
+
icont = 0
|
|
469
|
+
if len(cytomicobj.lst_attuuid) > 0:
|
|
470
|
+
for uuid in cytomicobj.lst_attuuid:
|
|
471
|
+
attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True)
|
|
472
|
+
if len(attr.tags) > 0:
|
|
473
|
+
for tg in attr.tags:
|
|
474
|
+
if tg.name == cytomicobj.tag:
|
|
475
|
+
cytomicobj.misp.untag(uuid, cytomicobj.tag)
|
|
476
|
+
icont += 1
|
|
477
|
+
break
|
|
478
|
+
print('Attributes untagged (' + str(icont) + ')')
|
|
479
|
+
except Exception:
|
|
480
|
+
cytomicobj.error = True
|
|
481
|
+
if cytomicobj.debug:
|
|
482
|
+
sys.exit('Unable to untag attributes')
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def process_attributes_upload(cytomicobj, moduleconfig):
|
|
486
|
+
# get attributes of each known type / prepare data to send / send data to Cytomic
|
|
487
|
+
try:
|
|
488
|
+
collect_events_ids(cytomicobj, moduleconfig)
|
|
489
|
+
process_attributes(cytomicobj, moduleconfig)
|
|
490
|
+
except Exception:
|
|
491
|
+
cytomicobj.error = True
|
|
492
|
+
if cytomicobj.debug:
|
|
493
|
+
sys.exit('Unable to upload attributes to Cytomic')
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def process_attributes_delete(cytomicobj, moduleconfig):
|
|
497
|
+
# get attributes of each known type / prepare data to send / send data to Cytomic
|
|
498
|
+
try:
|
|
499
|
+
collect_events_ids(cytomicobj, moduleconfig)
|
|
500
|
+
process_attributes(cytomicobj, moduleconfig)
|
|
501
|
+
except Exception:
|
|
502
|
+
cytomicobj.error = True
|
|
503
|
+
if cytomicobj.debug:
|
|
504
|
+
sys.exit('Unable to delete attributes in Cytomic')
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
if __name__ == '__main__':
|
|
508
|
+
parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API')
|
|
509
|
+
group = parser.add_mutually_exclusive_group()
|
|
510
|
+
group.add_argument('--events', action='store_true', help='Upload events indicators')
|
|
511
|
+
group.add_argument('--upload', action='store_true', help='Upload indicators')
|
|
512
|
+
group.add_argument('--delete', action='store_true', help='Delete indicators')
|
|
513
|
+
args = parser.parse_args()
|
|
514
|
+
if not args.upload and not args.delete and not args.events:
|
|
515
|
+
sys.exit("No valid action for the API")
|
|
516
|
+
|
|
517
|
+
if misp_verifycert is False:
|
|
518
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
519
|
+
|
|
520
|
+
module_config = get_config(misp_url, misp_key, misp_verifycert)
|
|
521
|
+
cytomicobj = cytomicobject
|
|
522
|
+
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug)
|
|
523
|
+
|
|
524
|
+
cytomicobj.misp = misp
|
|
525
|
+
cytomicobj.args = args
|
|
526
|
+
|
|
527
|
+
access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password'])
|
|
528
|
+
cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token}
|
|
529
|
+
if cytomicobj.debug:
|
|
530
|
+
print('Received access token')
|
|
531
|
+
|
|
532
|
+
if cytomicobj.args.events:
|
|
533
|
+
cytomicobj.tag = module_config['upload_tag']
|
|
534
|
+
cytomicobj.limit_events = module_config['limit_upload_events']
|
|
535
|
+
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
|
536
|
+
process_events(cytomicobj, module_config)
|
|
537
|
+
print_result_events(cytomicobj)
|
|
538
|
+
|
|
539
|
+
elif cytomicobj.args.upload:
|
|
540
|
+
cytomicobj.tag = module_config['upload_tag']
|
|
541
|
+
cytomicobj.limit_events = 0
|
|
542
|
+
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
|
543
|
+
process_attributes_upload(cytomicobj, module_config)
|
|
544
|
+
|
|
545
|
+
else:
|
|
546
|
+
cytomicobj.tag = module_config['delete_tag']
|
|
547
|
+
cytomicobj.limit_events = 0
|
|
548
|
+
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
|
549
|
+
process_attributes_delete(cytomicobj, module_config)
|
examples/del.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
from pymisp import ExpandedPyMISP
|
|
5
|
+
from keys import misp_url, misp_key, misp_verifycert
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
if __name__ == '__main__':
|
|
10
|
+
parser = argparse.ArgumentParser(description='Delete an event from a MISP instance.')
|
|
11
|
+
parser.add_argument("-e", "--event", help="Event ID to delete.")
|
|
12
|
+
parser.add_argument("-a", "--attribute", help="Attribute ID to delete.")
|
|
13
|
+
|
|
14
|
+
args = parser.parse_args()
|
|
15
|
+
|
|
16
|
+
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
|
17
|
+
|
|
18
|
+
if args.event:
|
|
19
|
+
result = misp.delete_event(args.event)
|
|
20
|
+
else:
|
|
21
|
+
result = misp.delete_attribute(args.attribute)
|
|
22
|
+
print(result)
|
examples/delete_user.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
from pymisp import ExpandedPyMISP
|
|
5
|
+
from keys import misp_url, misp_key, misp_verifycert
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
if __name__ == '__main__':
|
|
10
|
+
parser = argparse.ArgumentParser(description='Delete the user with the given id. Keep in mind that disabling users (by setting the disabled flag via an edit) is always preferred to keep user associations to events intact.')
|
|
11
|
+
parser.add_argument("-i", "--user_id", help="The id of the user you want to delete.")
|
|
12
|
+
args = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
|
15
|
+
|
|
16
|
+
print(misp.delete_user(args.user_id))
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
from pymisp import ExpandedPyMISP, MISPOrganisation
|
|
5
|
+
from keys import misp_url, misp_key, misp_verifycert
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
if __name__ == '__main__':
|
|
9
|
+
parser = argparse.ArgumentParser(description='Edit the email of the organisation designed by the organisation_id.')
|
|
10
|
+
parser.add_argument("-i", "--organisation_id", required=True, help="The name of the json file describing the organisation you want to modify.")
|
|
11
|
+
parser.add_argument("-e", "--email", help="Email linked to the organisation.")
|
|
12
|
+
args = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
|
15
|
+
|
|
16
|
+
org = MISPOrganisation()
|
|
17
|
+
org.id = args.organisation_id
|
|
18
|
+
org.email = args.email
|
|
19
|
+
|
|
20
|
+
print(misp.update_organisation(org, pythonify=True))
|
examples/edit_user.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
from pymisp import ExpandedPyMISP, MISPUser
|
|
5
|
+
from keys import misp_url, misp_key, misp_verifycert
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
if __name__ == '__main__':
|
|
10
|
+
parser = argparse.ArgumentParser(description='Edit the email of the user designed by the user_id.')
|
|
11
|
+
parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.")
|
|
12
|
+
parser.add_argument("-e", "--email", help="Email linked to the account.")
|
|
13
|
+
args = parser.parse_args()
|
|
14
|
+
|
|
15
|
+
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
|
16
|
+
user = MISPUser
|
|
17
|
+
user.id = args.user_id
|
|
18
|
+
user.email = args.email
|
|
19
|
+
|
|
20
|
+
print(misp.edit_user(user, pythonify=True))
|