pymisp 2.5.4__py3-none-any.whl → 2.5.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pymisp might be problematic. Click here for more details.

Files changed (158) hide show
  1. CHANGELOG.txt +5380 -0
  2. examples/__init__.py +0 -0
  3. examples/add_attributes_from_csv.py +74 -0
  4. examples/add_email_object.py +29 -0
  5. examples/add_fail2ban_object.py +86 -0
  6. examples/add_feed.py +25 -0
  7. examples/add_file_object.py +47 -0
  8. examples/add_filetype_object_from_csv.py +53 -0
  9. examples/add_generic_object.py +26 -0
  10. examples/add_github_user.py +65 -0
  11. examples/add_gitlab_user.py +56 -0
  12. examples/add_named_attribute.py +25 -0
  13. examples/add_organisations.py +57 -0
  14. examples/add_ssh_authorized_keys.py +29 -0
  15. examples/add_user.py +22 -0
  16. examples/add_vehicle_object.py +22 -0
  17. examples/addtag2.py +45 -0
  18. examples/asciidoc_generator.py +114 -0
  19. examples/cache_all.py +10 -0
  20. examples/copyTagsFromAttributesToEvent.py +68 -0
  21. examples/copy_list.py +93 -0
  22. examples/create_events.py +26 -0
  23. examples/cytomic_orion.py +549 -0
  24. examples/del.py +22 -0
  25. examples/delete_user.py +16 -0
  26. examples/edit_organisation.py +20 -0
  27. examples/edit_user.py +20 -0
  28. examples/falsepositive_disabletoids.py +136 -0
  29. examples/fetch_events_feed.py +15 -0
  30. examples/fetch_warninglist_hits.py +38 -0
  31. examples/freetext.py +22 -0
  32. examples/generate_file_objects.py +78 -0
  33. examples/generate_meta_feed.py +15 -0
  34. examples/get.py +37 -0
  35. examples/get_csv.py +37 -0
  36. examples/get_network_activity.py +187 -0
  37. examples/last.py +48 -0
  38. examples/load_csv.py +94 -0
  39. examples/lookup.py +28 -0
  40. examples/misp2cef.py +71 -0
  41. examples/misp2clamav.py +52 -0
  42. examples/openioc_to_misp.py +27 -0
  43. examples/proofpoint_tap.py +203 -0
  44. examples/proofpoint_vap.py +65 -0
  45. examples/search.py +48 -0
  46. examples/search_attributes_yara.py +40 -0
  47. examples/search_sighting.py +42 -0
  48. examples/server_sync_check_conn.py +32 -0
  49. examples/sharing_groups.py +15 -0
  50. examples/show_sightings.py +168 -0
  51. examples/stats_report.py +405 -0
  52. examples/sync_sighting.py +171 -0
  53. examples/tags.py +25 -0
  54. examples/test_sign.py +19 -0
  55. examples/trustar_misp.py +59 -0
  56. examples/up.py +21 -0
  57. examples/upload.py +60 -0
  58. examples/users_list.py +15 -0
  59. examples/vmray_automation.py +281 -0
  60. examples/vt_to_misp.py +182 -0
  61. examples/warninglists.py +22 -0
  62. examples/yara.py +38 -0
  63. examples/yara_dump.py +98 -0
  64. pymisp/api.py +33 -5
  65. pymisp/data/misp-objects/objects/instagram-account/definition.json +66 -0
  66. pymisp/data/misp-objects/objects/lnk/definition.json +13 -1
  67. pymisp/data/misp-objects/objects/rmm/definition.json +88 -0
  68. pymisp/data/misp-objects/objects/target-system/definition.json +2 -2
  69. pymisp/data/misp-objects/schema_objects.json +1 -1
  70. pymisp/mispevent.py +8 -0
  71. {pymisp-2.5.4.dist-info → pymisp-2.5.7.dist-info}/METADATA +23 -28
  72. {pymisp-2.5.4.dist-info → pymisp-2.5.7.dist-info}/RECORD +140 -27
  73. {pymisp-2.5.4.dist-info → pymisp-2.5.7.dist-info}/WHEEL +1 -1
  74. tests/57c4445b-c548-4654-af0b-4be3950d210f.json +1 -0
  75. tests/__init__.py +0 -0
  76. tests/csv_testfiles/invalid_fieldnames.csv +11 -0
  77. tests/csv_testfiles/valid_fieldnames.csv +4 -0
  78. tests/email_testfiles/mail_1.eml.zip +0 -0
  79. tests/email_testfiles/mail_1.msg +0 -0
  80. tests/email_testfiles/mail_1_bom.eml +858 -0
  81. tests/email_testfiles/mail_1_headers_only.eml +28 -0
  82. tests/email_testfiles/mail_2.eml +32 -0
  83. tests/email_testfiles/mail_3.eml +170 -0
  84. tests/email_testfiles/mail_3.msg +0 -0
  85. tests/email_testfiles/mail_4.msg +0 -0
  86. tests/email_testfiles/mail_5.msg +0 -0
  87. tests/email_testfiles/mail_multiple_to.eml +15 -0
  88. tests/email_testfiles/source +1 -0
  89. tests/git-vuln-finder-quagga.json +1493 -0
  90. tests/misp_event.json +76 -0
  91. tests/mispevent_testfiles/attribute.json +21 -0
  92. tests/mispevent_testfiles/attribute_del.json +23 -0
  93. tests/mispevent_testfiles/def_param.json +53 -0
  94. tests/mispevent_testfiles/event.json +8 -0
  95. tests/mispevent_testfiles/event_obj_attr_tag.json +57 -0
  96. tests/mispevent_testfiles/event_obj_def_param.json +62 -0
  97. tests/mispevent_testfiles/event_obj_tag.json +29 -0
  98. tests/mispevent_testfiles/event_tags.json +18 -0
  99. tests/mispevent_testfiles/existing_event.json +4599 -0
  100. tests/mispevent_testfiles/existing_event_edited.json +4601 -0
  101. tests/mispevent_testfiles/galaxy.json +25 -0
  102. tests/mispevent_testfiles/malware.json +19 -0
  103. tests/mispevent_testfiles/malware_exist.json +163 -0
  104. tests/mispevent_testfiles/misp_custom_obj.json +38 -0
  105. tests/mispevent_testfiles/overwrite_file/definition.json +457 -0
  106. tests/mispevent_testfiles/proposals.json +35 -0
  107. tests/mispevent_testfiles/shadow.json +148 -0
  108. tests/mispevent_testfiles/sighting.json +5 -0
  109. tests/mispevent_testfiles/simple.json +2 -0
  110. tests/mispevent_testfiles/test_object_template/definition.json +29 -0
  111. tests/new_misp_event.json +34 -0
  112. tests/reportlab_testfiles/HTML_event.json +1 -0
  113. tests/reportlab_testfiles/galaxy_1.json +1250 -0
  114. tests/reportlab_testfiles/image_event.json +2490 -0
  115. tests/reportlab_testfiles/japanese_test.json +156 -0
  116. tests/reportlab_testfiles/japanese_test_heavy.json +318 -0
  117. tests/reportlab_testfiles/long_event.json +3730 -0
  118. tests/reportlab_testfiles/mainly_objects_1.json +1092 -0
  119. tests/reportlab_testfiles/mainly_objects_2.json +977 -0
  120. tests/reportlab_testfiles/sighting_1.json +305 -0
  121. tests/reportlab_testfiles/sighting_2.json +221 -0
  122. tests/reportlab_testfiles/to_delete1.json +804 -0
  123. tests/reportlab_testfiles/to_delete2.json +1 -0
  124. tests/reportlab_testfiles/to_delete3.json +1 -0
  125. tests/reportlab_testfiles/very_long_event.json +1006 -0
  126. tests/reportlab_testoutputs/to_delete1.json.pdf +391 -0
  127. tests/reportlab_testoutputs/to_delete2.json.pdf +506 -0
  128. tests/reportlab_testoutputs/to_delete3.json.pdf +277 -0
  129. tests/search_index_result.json +69 -0
  130. tests/sharing_groups.json +98 -0
  131. tests/stix1.xml-utf8 +110 -0
  132. tests/stix2.json +1 -0
  133. tests/test_analyst_data.py +123 -0
  134. tests/test_emailobject.py +157 -0
  135. tests/test_fileobject.py +20 -0
  136. tests/test_mispevent.py +473 -0
  137. tests/test_reportlab.py +431 -0
  138. tests/testlive_comprehensive.py +3734 -0
  139. tests/testlive_sync.py +474 -0
  140. pymisp/data/misp-objects/.git +0 -1
  141. pymisp/data/misp-objects/.gitchangelog.rc +0 -289
  142. pymisp/data/misp-objects/.github/workflows/codeql.yml +0 -41
  143. pymisp/data/misp-objects/.github/workflows/nosetests.yml +0 -39
  144. pymisp/data/misp-objects/.travis.yml +0 -16
  145. pymisp/data/misp-objects/LICENSE-software-only.md +0 -661
  146. pymisp/data/misp-objects/LICENSE.md +0 -36
  147. pymisp/data/misp-objects/README.md +0 -567
  148. pymisp/data/misp-objects/docs/time-related-objects.ods +0 -0
  149. pymisp/data/misp-objects/docs/time-related-objects.pdf +0 -0
  150. pymisp/data/misp-objects/jq_all_the_things.sh +0 -29
  151. pymisp/data/misp-objects/tools/adoc_objects.py +0 -145
  152. pymisp/data/misp-objects/tools/alfred_links_to_relarelationships.py +0 -48
  153. pymisp/data/misp-objects/tools/list_of_objects.py +0 -50
  154. pymisp/data/misp-objects/tools/updated.sh +0 -6
  155. pymisp/data/misp-objects/tools/validate_opposites.sh +0 -17
  156. pymisp/data/misp-objects/unique_uuid.py +0 -16
  157. pymisp/data/misp-objects/validate_all.sh +0 -38
  158. {pymisp-2.5.4.dist-info → pymisp-2.5.7.dist-info}/LICENSE +0 -0
@@ -0,0 +1,549 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ '''
5
+ Koen Van Impe
6
+
7
+ Cytomic Automation
8
+ Put this script in crontab to run every /15 or /60
9
+ */15 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py
10
+
11
+
12
+ Fetches the configuration set in the Cytomic Orion enrichment module
13
+ - events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion
14
+ - upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
15
+ - delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
16
+
17
+ '''
18
+
19
+ from pymisp import ExpandedPyMISP
20
+ from keys import misp_url, misp_key, misp_verifycert
21
+ import argparse
22
+ import os
23
+ import re
24
+ import sys
25
+ import requests
26
+ import json
27
+ import urllib3
28
+
29
+
30
+ def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password):
31
+ '''
32
+ Get oAuth2 token
33
+ Configuration settings are fetched first from the MISP module configu
34
+ '''
35
+
36
+ try:
37
+ if scope and grant_type and username and password:
38
+ data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password}
39
+
40
+ if token_url and clientid and clientsecret:
41
+ access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret))
42
+ tokens = json.loads(access_token_response.text)
43
+ if 'access_token' in tokens:
44
+ access_token = tokens['access_token']
45
+ return access_token
46
+ else:
47
+ sys.exit('No token received')
48
+ else:
49
+ sys.exit('No token_url, clientid or clientsecret supplied')
50
+ else:
51
+ sys.exit('No scope, grant_type, username or password supplied')
52
+ except Exception:
53
+ sys.exit('Unable to connect to token_url')
54
+
55
+
56
+ def get_config(url, key, misp_verifycert):
57
+ '''
58
+ Get the module config and the settings needed to access the API
59
+ Also contains the settings to do the query
60
+ '''
61
+ try:
62
+ misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
63
+ req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
64
+ if req.status_code == 200:
65
+ req_json = req.json()
66
+ if 'finalSettings' in req_json:
67
+ finalSettings = req_json['finalSettings']
68
+
69
+ clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = ''
70
+ module_enabled = False
71
+ scope = 'orion.api'
72
+ grant_type = 'password'
73
+ limit_upload_events = 50
74
+ limit_upload_attributes = 50
75
+ ttlDays = "1"
76
+ last_attributes = '5d'
77
+ post_threat_level_id = 2
78
+ for el in finalSettings:
79
+ # Is the module enabled?
80
+ if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled':
81
+ module_enabled = el['value']
82
+ if module_enabled is False:
83
+ break
84
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid':
85
+ clientid = el['value']
86
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret':
87
+ clientsecret = el['value']
88
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username':
89
+ username = el['value']
90
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password':
91
+ password = el['value']
92
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url':
93
+ api_url = el['value'].replace('\\/', '/')
94
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url':
95
+ token_url = el['value'].replace('\\/', '/')
96
+ elif el['setting'] == 'MISP.baseurl':
97
+ misp_baseurl = el['value']
98
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id':
99
+ if el['value']:
100
+ try:
101
+ post_threat_level_id = int(el['value'])
102
+ except:
103
+ continue
104
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays':
105
+ if el['value']:
106
+ try:
107
+ ttlDays = "{last_days}".format(last_days=int(el['value']))
108
+ except:
109
+ continue
110
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe':
111
+ if el['value']:
112
+ try:
113
+ last_attributes = "{last_days}d".format(last_days=int(el['value']))
114
+ except:
115
+ continue
116
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag':
117
+ upload_tag = el['value']
118
+ elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag':
119
+ delete_tag = el['value']
120
+ elif el['setting'] == 'Plugin.Enrichment_limit_upload_events':
121
+ if el['value']:
122
+ try:
123
+ limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value']))
124
+ except:
125
+ continue
126
+ elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes':
127
+ if el['value']:
128
+ try:
129
+ limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value']))
130
+ except:
131
+ continue
132
+ else:
133
+ sys.exit('Did not receive a 200 code from MISP')
134
+
135
+ if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type:
136
+
137
+ return {'cytomic_policy': 'Detect',
138
+ 'upload_timeframe': last_attributes,
139
+ 'upload_tag': upload_tag,
140
+ 'delete_tag': delete_tag,
141
+ 'upload_ttlDays': ttlDays,
142
+ 'post_threat_level_id': post_threat_level_id,
143
+ 'clientid': clientid,
144
+ 'clientsecret': clientsecret,
145
+ 'scope': scope,
146
+ 'username': username,
147
+ 'password': password,
148
+ 'grant_type': grant_type,
149
+ 'api_url': api_url,
150
+ 'token_url': token_url,
151
+ 'misp_baseurl': misp_baseurl,
152
+ 'limit_upload_events': limit_upload_events,
153
+ 'limit_upload_attributes': limit_upload_attributes}
154
+ else:
155
+ sys.exit('Did not receive all the necessary configuration information from MISP')
156
+
157
+ except Exception as e:
158
+ sys.exit('Unable to get module config from MISP')
159
+
160
+
161
+ class cytomicobject:
162
+ misp = None
163
+ lst_evtid = None
164
+ lst_attuuid = None
165
+ lst_attuuid_error = None
166
+ endpoint_ioc = None
167
+ api_call_headers = None
168
+ post_data = None
169
+ args = None
170
+ tag = None
171
+ limit_events = None
172
+ limit_attributes = None
173
+ atttype_misp = None
174
+ atttype_cytomic = None
175
+ attlabel_cytomic = None
176
+ att_types = {
177
+ "ip-dst": {"ip": "ipioc"},
178
+ "ip-src": {"ip": "ipioc"},
179
+ "url": {"url": "urlioc"},
180
+ "md5": {"hash": "filehashioc"},
181
+ "domain": {"domain": "domainioc"},
182
+ "hostname": {"domain": "domainioc"},
183
+ "domain|ip": {"domain": "domainioc"},
184
+ "hostname|port": {"domain": "domainioc"}
185
+ }
186
+ debug = True
187
+ error = False
188
+ res = False
189
+ res_msg = None
190
+
191
+
192
+ def collect_events_ids(cytomicobj, moduleconfig):
193
+ # Get events that contain Cytomic tag.
194
+ try:
195
+ evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True)
196
+ cytomicobj.lst_evtid = ['x', 'y']
197
+ for evt in evt_result:
198
+ evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True)
199
+ if len(evt.tags) > 0:
200
+ for tg in evt.tags:
201
+ if tg.name == cytomicobj.tag:
202
+ if not cytomicobj.lst_evtid:
203
+ cytomicobj.lst_evtid = str(evt['id'])
204
+ else:
205
+ if not evt['id'] in cytomicobj.lst_evtid:
206
+ cytomicobj.lst_evtid.append(str(evt['id']))
207
+ break
208
+ cytomicobj.lst_evtid.remove('x')
209
+ cytomicobj.lst_evtid.remove('y')
210
+ except Exception:
211
+ cytomicobj.error = True
212
+ if cytomicobj.debug:
213
+ sys.exit('Unable to collect events ids')
214
+
215
+
216
+ def find_eventid(cytomicobj, evtid):
217
+ # Get events that contain Cytomic tag.
218
+ try:
219
+ cytomicobj.res = False
220
+ for id in cytomicobj.lst_evtid:
221
+ if id == evtid:
222
+ cytomicobj.res = True
223
+ break
224
+ except Exception:
225
+ cytomicobj.error = True
226
+ if cytomicobj.debug:
227
+ sys.exit('Unable to collect events ids')
228
+
229
+
230
+ def print_result_events(cytomicobj):
231
+ try:
232
+ if cytomicobj.res_msg is not None:
233
+ for key, msg in cytomicobj.res_msg.items():
234
+ if msg is not None:
235
+ print(key, msg)
236
+ except Exception:
237
+ cytomicobj.error = True
238
+ if cytomicobj.debug:
239
+ sys.exit('Unable to print result')
240
+
241
+
242
+ def set_postdata(cytomicobj, moduleconfig, attribute):
243
+ # Set JSON to send to the API.
244
+ try:
245
+
246
+ if cytomicobj.args.upload or cytomicobj.args.events:
247
+ event = attribute['Event']
248
+ event_title = event['info']
249
+ event_id = event['id']
250
+ threat_level_id = int(event['threat_level_id'])
251
+ if moduleconfig['post_threat_level_id'] <= threat_level_id:
252
+
253
+ if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
254
+ post_value = attribute['value'].split('|')[0]
255
+ else:
256
+ post_value = attribute['value']
257
+
258
+ if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
259
+ pass
260
+ else:
261
+ if cytomicobj.post_data is None:
262
+ cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}]
263
+ else:
264
+ if post_value not in str(cytomicobj.post_data):
265
+ cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()})
266
+ else:
267
+ if cytomicobject.debug:
268
+ print('Event %s skipped because of lower threat level' % event_id)
269
+ else:
270
+ event = attribute['Event']
271
+ threat_level_id = int(event['threat_level_id'])
272
+ if moduleconfig['post_threat_level_id'] <= threat_level_id:
273
+ if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
274
+ post_value = attribute['value'].split('|')[0]
275
+ else:
276
+ post_value = attribute['value']
277
+
278
+ if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
279
+ pass
280
+ else:
281
+ if cytomicobj.post_data is None:
282
+ cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}]
283
+ else:
284
+ cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value})
285
+ else:
286
+ if cytomicobject.debug:
287
+ print('Event %s skipped because of lower threat level' % event_id)
288
+ except Exception:
289
+ cytomicobj.error = True
290
+ if cytomicobj.debug:
291
+ sys.exit('Unable to process post-data')
292
+
293
+
294
+ def send_postdata(cytomicobj, evtid=None):
295
+ # Batch post to upload event attributes.
296
+ try:
297
+ if cytomicobj.post_data is not None:
298
+ if cytomicobj.debug:
299
+ print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data))
300
+ result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False)
301
+ json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text)
302
+ print(result_post_endpoint_ioc)
303
+ if 'true' not in (result_post_endpoint_ioc.text):
304
+ cytomicobj.error = True
305
+ if evtid is not None:
306
+ if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
307
+ cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
308
+ else:
309
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
310
+ if cytomicobj.debug:
311
+ print('RESULT: {}'.format(json_result_post_endpoint_ioc))
312
+ else:
313
+ if evtid is None:
314
+ cytomicobj.error = True
315
+ except Exception:
316
+ cytomicobj.error = True
317
+ if cytomicobj.debug:
318
+ sys.exit('Unable to post attributes')
319
+
320
+
321
+ def process_attributes(cytomicobj, moduleconfig, evtid=None):
322
+ # Get attributes to process.
323
+ try:
324
+ for misptype, cytomictypes in cytomicobject.att_types.items():
325
+ cytomicobj.atttype_misp = misptype
326
+ for cytomiclabel, cytomictype in cytomictypes.items():
327
+ cytomicobj.attlabel_cytomic = cytomiclabel
328
+ cytomicobj.atttype_cytomic = cytomictype
329
+ cytomicobj.post_data = None
330
+ icont = 0
331
+ if cytomicobj.args.upload or cytomicobj.args.events:
332
+ cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays'])
333
+ else:
334
+ cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic
335
+
336
+ # Get attributes to upload/delete and prepare JSON
337
+ # If evtid is set; we're called from --events
338
+ if cytomicobject.debug:
339
+ print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp)
340
+
341
+ if evtid is None:
342
+ cytomicobj.error = False
343
+ attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
344
+ else:
345
+ if cytomicobj.error:
346
+ break
347
+ # We don't search with tags; we have an event for which we want to upload all events
348
+ attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
349
+
350
+ cytomicobj.lst_attuuid = ['x', 'y']
351
+
352
+ if len(attr_result['Attribute']) > 0:
353
+ for attribute in attr_result['Attribute']:
354
+ if evtid is not None:
355
+ if cytomicobj.error:
356
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
357
+ break
358
+ if icont >= cytomicobj.limit_attributes:
359
+ if not cytomicobj.error and cytomicobj.post_data is not None:
360
+ # Send data to Cytomic
361
+ send_postdata(cytomicobj, evtid)
362
+ if not cytomicobj.error:
363
+ if 'Event: ' + str(evtid) in cytomicobj.res_msg:
364
+ if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
365
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
366
+ else:
367
+ cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
368
+ else:
369
+ if cytomicobject.debug:
370
+ print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
371
+
372
+ cytomicobj.post_data = None
373
+ if cytomicobj.error:
374
+ if evtid is not None:
375
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
376
+ break
377
+ icont = 0
378
+
379
+ if evtid is None:
380
+ event = attribute['Event']
381
+ event_id = event['id']
382
+ find_eventid(cytomicobj, str(event_id))
383
+ if not cytomicobj.res:
384
+ if not cytomicobj.lst_attuuid:
385
+ cytomicobj.lst_attuuid = attribute['uuid']
386
+ else:
387
+ if not attribute['uuid'] in cytomicobj.lst_attuuid:
388
+ cytomicobj.lst_attuuid.append(attribute['uuid'])
389
+ icont += 1
390
+ # Prepare data to send
391
+ set_postdata(cytomicobj, moduleconfig, attribute)
392
+ else:
393
+ icont += 1
394
+ # Prepare data to send
395
+ set_postdata(cytomicobj, moduleconfig, attribute)
396
+
397
+ if not cytomicobj.error:
398
+ # Send data to Cytomic
399
+ send_postdata(cytomicobj, evtid)
400
+
401
+ if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0:
402
+ # Data sent; process response
403
+ if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg:
404
+ if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
405
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
406
+ else:
407
+ cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
408
+ else:
409
+ if cytomicobject.debug:
410
+ print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
411
+
412
+ if not cytomicobj.error:
413
+ cytomicobj.lst_attuuid.remove('x')
414
+ cytomicobj.lst_attuuid.remove('y')
415
+ # Untag attributes
416
+ untag_attributes(cytomicobj)
417
+ except Exception:
418
+ cytomicobj.error = True
419
+ if cytomicobj.debug:
420
+ sys.exit('Unable to get attributes')
421
+
422
+
423
+ def untag_event(evtid):
424
+ # Remove tag of the event being processed.
425
+ try:
426
+ cytomicobj.records = 0
427
+ evt = cytomicobj.misp.get_event(event=evtid, pythonify=True)
428
+ if len(evt.tags) > 0:
429
+ for tg in evt.tags:
430
+ if tg.name == cytomicobj.tag:
431
+ cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag)
432
+ cytomicobj.records += 1
433
+ cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)'
434
+ break
435
+ except Exception:
436
+ cytomicobj.error = True
437
+ if cytomicobj.debug:
438
+ sys.exit('Unable to untag events')
439
+
440
+
441
+ def process_events(cytomicobj, moduleconfig):
442
+ # Get events that contain Cytomic tag.
443
+ try:
444
+ collect_events_ids(cytomicobj, moduleconfig)
445
+ total_attributes_sent = 0
446
+ for evtid in cytomicobj.lst_evtid:
447
+ cytomicobj.error = False
448
+ if cytomicobj.res_msg is None:
449
+ cytomicobj.res_msg = {'Event: ' + str(evtid): None}
450
+ else:
451
+ cytomicobj.res_msg['Event: ' + str(evtid)] = None
452
+ if cytomicobject.debug:
453
+ print('Event id: ' + str(evtid))
454
+
455
+ # get attributes of each known type of the event / prepare data to send / send data to Cytomic
456
+ process_attributes(cytomicobj, moduleconfig, evtid)
457
+ if not cytomicobj.error:
458
+ untag_event(evtid)
459
+ except Exception:
460
+ cytomicobj.error = True
461
+ if cytomicobj.debug:
462
+ sys.exit('Unable to process events ids')
463
+
464
+
465
+ def untag_attributes(cytomicobj):
466
+ # Remove tag of attributes sent.
467
+ try:
468
+ icont = 0
469
+ if len(cytomicobj.lst_attuuid) > 0:
470
+ for uuid in cytomicobj.lst_attuuid:
471
+ attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True)
472
+ if len(attr.tags) > 0:
473
+ for tg in attr.tags:
474
+ if tg.name == cytomicobj.tag:
475
+ cytomicobj.misp.untag(uuid, cytomicobj.tag)
476
+ icont += 1
477
+ break
478
+ print('Attributes untagged (' + str(icont) + ')')
479
+ except Exception:
480
+ cytomicobj.error = True
481
+ if cytomicobj.debug:
482
+ sys.exit('Unable to untag attributes')
483
+
484
+
485
+ def process_attributes_upload(cytomicobj, moduleconfig):
486
+ # get attributes of each known type / prepare data to send / send data to Cytomic
487
+ try:
488
+ collect_events_ids(cytomicobj, moduleconfig)
489
+ process_attributes(cytomicobj, moduleconfig)
490
+ except Exception:
491
+ cytomicobj.error = True
492
+ if cytomicobj.debug:
493
+ sys.exit('Unable to upload attributes to Cytomic')
494
+
495
+
496
+ def process_attributes_delete(cytomicobj, moduleconfig):
497
+ # get attributes of each known type / prepare data to send / send data to Cytomic
498
+ try:
499
+ collect_events_ids(cytomicobj, moduleconfig)
500
+ process_attributes(cytomicobj, moduleconfig)
501
+ except Exception:
502
+ cytomicobj.error = True
503
+ if cytomicobj.debug:
504
+ sys.exit('Unable to delete attributes in Cytomic')
505
+
506
+
507
+ if __name__ == '__main__':
508
+ parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API')
509
+ group = parser.add_mutually_exclusive_group()
510
+ group.add_argument('--events', action='store_true', help='Upload events indicators')
511
+ group.add_argument('--upload', action='store_true', help='Upload indicators')
512
+ group.add_argument('--delete', action='store_true', help='Delete indicators')
513
+ args = parser.parse_args()
514
+ if not args.upload and not args.delete and not args.events:
515
+ sys.exit("No valid action for the API")
516
+
517
+ if misp_verifycert is False:
518
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
519
+
520
+ module_config = get_config(misp_url, misp_key, misp_verifycert)
521
+ cytomicobj = cytomicobject
522
+ misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug)
523
+
524
+ cytomicobj.misp = misp
525
+ cytomicobj.args = args
526
+
527
+ access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password'])
528
+ cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token}
529
+ if cytomicobj.debug:
530
+ print('Received access token')
531
+
532
+ if cytomicobj.args.events:
533
+ cytomicobj.tag = module_config['upload_tag']
534
+ cytomicobj.limit_events = module_config['limit_upload_events']
535
+ cytomicobj.limit_attributes = module_config['limit_upload_attributes']
536
+ process_events(cytomicobj, module_config)
537
+ print_result_events(cytomicobj)
538
+
539
+ elif cytomicobj.args.upload:
540
+ cytomicobj.tag = module_config['upload_tag']
541
+ cytomicobj.limit_events = 0
542
+ cytomicobj.limit_attributes = module_config['limit_upload_attributes']
543
+ process_attributes_upload(cytomicobj, module_config)
544
+
545
+ else:
546
+ cytomicobj.tag = module_config['delete_tag']
547
+ cytomicobj.limit_events = 0
548
+ cytomicobj.limit_attributes = module_config['limit_upload_attributes']
549
+ process_attributes_delete(cytomicobj, module_config)
examples/del.py ADDED
@@ -0,0 +1,22 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from pymisp import ExpandedPyMISP
5
+ from keys import misp_url, misp_key, misp_verifycert
6
+ import argparse
7
+
8
+
9
+ if __name__ == '__main__':
10
+ parser = argparse.ArgumentParser(description='Delete an event from a MISP instance.')
11
+ parser.add_argument("-e", "--event", help="Event ID to delete.")
12
+ parser.add_argument("-a", "--attribute", help="Attribute ID to delete.")
13
+
14
+ args = parser.parse_args()
15
+
16
+ misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
17
+
18
+ if args.event:
19
+ result = misp.delete_event(args.event)
20
+ else:
21
+ result = misp.delete_attribute(args.attribute)
22
+ print(result)
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from pymisp import ExpandedPyMISP
5
+ from keys import misp_url, misp_key, misp_verifycert
6
+ import argparse
7
+
8
+
9
+ if __name__ == '__main__':
10
+ parser = argparse.ArgumentParser(description='Delete the user with the given id. Keep in mind that disabling users (by setting the disabled flag via an edit) is always preferred to keep user associations to events intact.')
11
+ parser.add_argument("-i", "--user_id", help="The id of the user you want to delete.")
12
+ args = parser.parse_args()
13
+
14
+ misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
15
+
16
+ print(misp.delete_user(args.user_id))
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from pymisp import ExpandedPyMISP, MISPOrganisation
5
+ from keys import misp_url, misp_key, misp_verifycert
6
+ import argparse
7
+
8
+ if __name__ == '__main__':
9
+ parser = argparse.ArgumentParser(description='Edit the email of the organisation designed by the organisation_id.')
10
+ parser.add_argument("-i", "--organisation_id", required=True, help="The name of the json file describing the organisation you want to modify.")
11
+ parser.add_argument("-e", "--email", help="Email linked to the organisation.")
12
+ args = parser.parse_args()
13
+
14
+ misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
15
+
16
+ org = MISPOrganisation()
17
+ org.id = args.organisation_id
18
+ org.email = args.email
19
+
20
+ print(misp.update_organisation(org, pythonify=True))
examples/edit_user.py ADDED
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from pymisp import ExpandedPyMISP, MISPUser
5
+ from keys import misp_url, misp_key, misp_verifycert
6
+ import argparse
7
+
8
+
9
+ if __name__ == '__main__':
10
+ parser = argparse.ArgumentParser(description='Edit the email of the user designed by the user_id.')
11
+ parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.")
12
+ parser.add_argument("-e", "--email", help="Email linked to the account.")
13
+ args = parser.parse_args()
14
+
15
+ misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
16
+ user = MISPUser
17
+ user.id = args.user_id
18
+ user.email = args.email
19
+
20
+ print(misp.edit_user(user, pythonify=True))