misp-modules 2.4.196__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. misp_modules/__init__.py +379 -0
  2. misp_modules/helpers/__init__.py +1 -0
  3. misp_modules/helpers/cache.py +84 -0
  4. misp_modules/lib/__init__.py +4 -0
  5. misp_modules/lib/_vmray/__init__.py +0 -0
  6. misp_modules/lib/_vmray/parser.py +1417 -0
  7. misp_modules/lib/_vmray/rest_api.py +148 -0
  8. misp_modules/lib/cof2misp/LICENSE-2.0.txt +202 -0
  9. misp_modules/lib/cof2misp/__init__.py +0 -0
  10. misp_modules/lib/cof2misp/cof.py +165 -0
  11. misp_modules/lib/joe_mapping.py +114 -0
  12. misp_modules/lib/joe_parser.py +573 -0
  13. misp_modules/lib/lastline_api.py +841 -0
  14. misp_modules/lib/qintel_helper.py +263 -0
  15. misp_modules/lib/stix2misp.py +2080 -0
  16. misp_modules/lib/stix2misp_mapping.py +460 -0
  17. misp_modules/lib/synonymsToTagNames.json +1 -0
  18. misp_modules/lib/vt_graph_parser/__init__.py +8 -0
  19. misp_modules/lib/vt_graph_parser/errors.py +20 -0
  20. misp_modules/lib/vt_graph_parser/helpers/__init__.py +7 -0
  21. misp_modules/lib/vt_graph_parser/helpers/parsers.py +88 -0
  22. misp_modules/lib/vt_graph_parser/helpers/rules.py +304 -0
  23. misp_modules/lib/vt_graph_parser/helpers/wrappers.py +58 -0
  24. misp_modules/lib/vt_graph_parser/importers/__init__.py +7 -0
  25. misp_modules/lib/vt_graph_parser/importers/base.py +98 -0
  26. misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +73 -0
  27. misp_modules/modules/__init__.py +4 -0
  28. misp_modules/modules/action_mod/__init__.py +1 -0
  29. misp_modules/modules/action_mod/_utils/__init__.py +1 -0
  30. misp_modules/modules/action_mod/_utils/utils.py +70 -0
  31. misp_modules/modules/action_mod/mattermost.py +113 -0
  32. misp_modules/modules/action_mod/slack.py +96 -0
  33. misp_modules/modules/action_mod/testaction.py +68 -0
  34. misp_modules/modules/expansion/__init__.py +36 -0
  35. misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT +27 -0
  36. misp_modules/modules/expansion/_dnsdb_query/LICENSE +202 -0
  37. misp_modules/modules/expansion/_dnsdb_query/README.md +162 -0
  38. misp_modules/modules/expansion/_dnsdb_query/__init__.py +0 -0
  39. misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +327 -0
  40. misp_modules/modules/expansion/_ransomcoindb/__init__.py +0 -0
  41. misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py +96 -0
  42. misp_modules/modules/expansion/_vulnerability_parser/__init__.py +0 -0
  43. misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py +118 -0
  44. misp_modules/modules/expansion/abuseipdb.py +146 -0
  45. misp_modules/modules/expansion/apiosintds.py +365 -0
  46. misp_modules/modules/expansion/apivoid.py +133 -0
  47. misp_modules/modules/expansion/assemblyline_query.py +179 -0
  48. misp_modules/modules/expansion/assemblyline_submit.py +100 -0
  49. misp_modules/modules/expansion/backscatter_io.py +84 -0
  50. misp_modules/modules/expansion/btc_scam_check.py +54 -0
  51. misp_modules/modules/expansion/btc_steroids.py +238 -0
  52. misp_modules/modules/expansion/censys_enrich.py +287 -0
  53. misp_modules/modules/expansion/circl_passivedns.py +85 -0
  54. misp_modules/modules/expansion/circl_passivessl.py +113 -0
  55. misp_modules/modules/expansion/clamav.py +129 -0
  56. misp_modules/modules/expansion/cluster25_expand.py +239 -0
  57. misp_modules/modules/expansion/countrycode.py +69 -0
  58. misp_modules/modules/expansion/cpe.py +140 -0
  59. misp_modules/modules/expansion/crowdsec.py +235 -0
  60. misp_modules/modules/expansion/crowdstrike_falcon.py +148 -0
  61. misp_modules/modules/expansion/cuckoo_submit.py +161 -0
  62. misp_modules/modules/expansion/cve.py +57 -0
  63. misp_modules/modules/expansion/cve_advanced.py +185 -0
  64. misp_modules/modules/expansion/cytomic_orion.py +196 -0
  65. misp_modules/modules/expansion/dbl_spamhaus.py +78 -0
  66. misp_modules/modules/expansion/dns.py +71 -0
  67. misp_modules/modules/expansion/docx_enrich.py +71 -0
  68. misp_modules/modules/expansion/domaintools.py +290 -0
  69. misp_modules/modules/expansion/eql.py +91 -0
  70. misp_modules/modules/expansion/eupi.py +86 -0
  71. misp_modules/modules/expansion/extract_url_components.py +83 -0
  72. misp_modules/modules/expansion/farsight_passivedns.py +243 -0
  73. misp_modules/modules/expansion/geoip_asn.py +75 -0
  74. misp_modules/modules/expansion/geoip_city.py +75 -0
  75. misp_modules/modules/expansion/geoip_country.py +73 -0
  76. misp_modules/modules/expansion/google_safe_browsing.py +86 -0
  77. misp_modules/modules/expansion/google_search.py +57 -0
  78. misp_modules/modules/expansion/google_threat_intelligence.py +453 -0
  79. misp_modules/modules/expansion/greynoise.py +346 -0
  80. misp_modules/modules/expansion/hashdd.py +53 -0
  81. misp_modules/modules/expansion/hashlookup.py +118 -0
  82. misp_modules/modules/expansion/hibp.py +62 -0
  83. misp_modules/modules/expansion/html_to_markdown.py +63 -0
  84. misp_modules/modules/expansion/hyasinsight.py +881 -0
  85. misp_modules/modules/expansion/intel471.py +73 -0
  86. misp_modules/modules/expansion/intelmq_eventdb.py.experimental +67 -0
  87. misp_modules/modules/expansion/ip2locationio.py +90 -0
  88. misp_modules/modules/expansion/ipasn.py +70 -0
  89. misp_modules/modules/expansion/ipinfo.py +112 -0
  90. misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py +633 -0
  91. misp_modules/modules/expansion/iprep.py +98 -0
  92. misp_modules/modules/expansion/jinja_template_rendering.py +54 -0
  93. misp_modules/modules/expansion/joesandbox_query.py +91 -0
  94. misp_modules/modules/expansion/joesandbox_submit.py +147 -0
  95. misp_modules/modules/expansion/lastline_query.py +148 -0
  96. misp_modules/modules/expansion/lastline_submit.py +180 -0
  97. misp_modules/modules/expansion/macaddress_io.py +130 -0
  98. misp_modules/modules/expansion/macvendors.py +55 -0
  99. misp_modules/modules/expansion/malshare_upload.py +107 -0
  100. misp_modules/modules/expansion/malwarebazaar.py +67 -0
  101. misp_modules/modules/expansion/mcafee_insights_enrich.py +249 -0
  102. misp_modules/modules/expansion/mmdb_lookup.py +138 -0
  103. misp_modules/modules/expansion/module.py.skeleton +44 -0
  104. misp_modules/modules/expansion/mwdb.py +152 -0
  105. misp_modules/modules/expansion/ocr_enrich.py +74 -0
  106. misp_modules/modules/expansion/ods_enrich.py +71 -0
  107. misp_modules/modules/expansion/odt_enrich.py +61 -0
  108. misp_modules/modules/expansion/onyphe.py +243 -0
  109. misp_modules/modules/expansion/onyphe_full.py +387 -0
  110. misp_modules/modules/expansion/otx.py +167 -0
  111. misp_modules/modules/expansion/passive_ssh.py +150 -0
  112. misp_modules/modules/expansion/passivetotal.py +358 -0
  113. misp_modules/modules/expansion/pdf_enrich.py +58 -0
  114. misp_modules/modules/expansion/pptx_enrich.py +65 -0
  115. misp_modules/modules/expansion/qintel_qsentry.py +229 -0
  116. misp_modules/modules/expansion/qrcode.py +99 -0
  117. misp_modules/modules/expansion/ransomcoindb.py +82 -0
  118. misp_modules/modules/expansion/rbl.py +126 -0
  119. misp_modules/modules/expansion/recordedfuture.py +544 -0
  120. misp_modules/modules/expansion/reversedns.py +77 -0
  121. misp_modules/modules/expansion/securitytrails.py +571 -0
  122. misp_modules/modules/expansion/shodan.py +244 -0
  123. misp_modules/modules/expansion/sigma_queries.py +61 -0
  124. misp_modules/modules/expansion/sigma_syntax_validator.py +49 -0
  125. misp_modules/modules/expansion/sigmf_expand.py +303 -0
  126. misp_modules/modules/expansion/socialscan.py +108 -0
  127. misp_modules/modules/expansion/sophoslabs_intelix.py +146 -0
  128. misp_modules/modules/expansion/sourcecache.py +57 -0
  129. misp_modules/modules/expansion/stairwell.py +156 -0
  130. misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +56 -0
  131. misp_modules/modules/expansion/threatcrowd.py +170 -0
  132. misp_modules/modules/expansion/threatfox.py +75 -0
  133. misp_modules/modules/expansion/threatminer.py +152 -0
  134. misp_modules/modules/expansion/triage_submit.py +119 -0
  135. misp_modules/modules/expansion/trustar_enrich.py +235 -0
  136. misp_modules/modules/expansion/urlhaus.py +164 -0
  137. misp_modules/modules/expansion/urlscan.py +269 -0
  138. misp_modules/modules/expansion/variotdbs.py +133 -0
  139. misp_modules/modules/expansion/virustotal.py +311 -0
  140. misp_modules/modules/expansion/virustotal_public.py +272 -0
  141. misp_modules/modules/expansion/virustotal_upload.py +91 -0
  142. misp_modules/modules/expansion/vmray_submit.py +161 -0
  143. misp_modules/modules/expansion/vmware_nsx.py +628 -0
  144. misp_modules/modules/expansion/vulndb.py +292 -0
  145. misp_modules/modules/expansion/vulnerability_lookup.py +324 -0
  146. misp_modules/modules/expansion/vulners.py +81 -0
  147. misp_modules/modules/expansion/vysion.py +221 -0
  148. misp_modules/modules/expansion/whois.py +72 -0
  149. misp_modules/modules/expansion/whoisfreaks.py +232 -0
  150. misp_modules/modules/expansion/wiki.py +58 -0
  151. misp_modules/modules/expansion/xforceexchange.py +189 -0
  152. misp_modules/modules/expansion/xlsx_enrich.py +63 -0
  153. misp_modules/modules/expansion/yara_query.py +70 -0
  154. misp_modules/modules/expansion/yara_syntax_validator.py +49 -0
  155. misp_modules/modules/expansion/yeti.py +196 -0
  156. misp_modules/modules/export_mod/__init__.py +3 -0
  157. misp_modules/modules/export_mod/cef_export.py +93 -0
  158. misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +150 -0
  159. misp_modules/modules/export_mod/defender_endpoint_export.py +141 -0
  160. misp_modules/modules/export_mod/goamlexport.py +255 -0
  161. misp_modules/modules/export_mod/liteexport.py +97 -0
  162. misp_modules/modules/export_mod/mass_eql_export.py +106 -0
  163. misp_modules/modules/export_mod/nexthinkexport.py +131 -0
  164. misp_modules/modules/export_mod/osqueryexport.py +125 -0
  165. misp_modules/modules/export_mod/pdfexport.py +107 -0
  166. misp_modules/modules/export_mod/testexport.py +73 -0
  167. misp_modules/modules/export_mod/threatStream_misp_export.py +114 -0
  168. misp_modules/modules/export_mod/threat_connect_export.py +125 -0
  169. misp_modules/modules/export_mod/virustotal_collections.py +141 -0
  170. misp_modules/modules/export_mod/vt_graph.py +120 -0
  171. misp_modules/modules/export_mod/yara_export.py +290 -0
  172. misp_modules/modules/import_mod/__init__.py +22 -0
  173. misp_modules/modules/import_mod/cof2misp.py +264 -0
  174. misp_modules/modules/import_mod/csvimport.py +319 -0
  175. misp_modules/modules/import_mod/cuckooimport.py +749 -0
  176. misp_modules/modules/import_mod/email_import.py +294 -0
  177. misp_modules/modules/import_mod/goamlimport.py +188 -0
  178. misp_modules/modules/import_mod/import_blueprint.py +93 -0
  179. misp_modules/modules/import_mod/joe_import.py +74 -0
  180. misp_modules/modules/import_mod/lastline_import.py +160 -0
  181. misp_modules/modules/import_mod/mispjson.py +73 -0
  182. misp_modules/modules/import_mod/ocr.py +122 -0
  183. misp_modules/modules/import_mod/openiocimport.py +99 -0
  184. misp_modules/modules/import_mod/taxii21.py +383 -0
  185. misp_modules/modules/import_mod/testimport.py +73 -0
  186. misp_modules/modules/import_mod/threatanalyzer_import.py +552 -0
  187. misp_modules/modules/import_mod/url_import.py +97 -0
  188. misp_modules/modules/import_mod/vmray_import.py +96 -0
  189. misp_modules/modules/import_mod/vmray_summary_json_import.py +87 -0
  190. misp_modules-2.4.196.dist-info/LICENSE +661 -0
  191. misp_modules-2.4.196.dist-info/METADATA +281 -0
  192. misp_modules-2.4.196.dist-info/RECORD +194 -0
  193. misp_modules-2.4.196.dist-info/WHEEL +4 -0
  194. misp_modules-2.4.196.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,2080 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
4
+ # Copyright (C) 2017-2018 Christian Studer
5
+ #
6
+ # This program is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU Affero General Public License as
8
+ # published by the Free Software Foundation, either version 3 of the
9
+ # License, or (at your option) any later version.
10
+ #
11
+ # This program is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU Affero General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU Affero General Public License
17
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+
19
+ import sys
20
+ import json
21
+ import os
22
+ import time
23
+ import io
24
+ import pymisp
25
+ import stix2misp_mapping
26
+ from collections import defaultdict
27
+ from copy import deepcopy
28
+ from pathlib import Path
29
+ _misp_dir = Path(os.path.realpath(__file__)).parents[4]
30
+ _misp_objects_path = _misp_dir / 'app' / 'files' / 'misp-objects' / 'objects'
31
+ _misp_types = pymisp.AbstractMISP().describe_types.get('types')
32
+ from pymisp import MISPEvent, MISPObject, MISPAttribute
33
+
34
+ _scripts_path = Path(__file__).resolve().parents[1]
35
+ sys.path.insert(0, str(_scripts_path / 'cti-python-stix2'))
36
+ import stix2
37
+
38
+
39
+ class StixParser():
40
+ _galaxy_types = ('intrusion-set', 'malware', 'threat-actor', 'tool')
41
+ _stix2misp_mapping = {'marking-definition': '_load_marking',
42
+ 'relationship': '_load_relationship',
43
+ 'report': '_load_report',
44
+ 'indicator': '_parse_indicator',
45
+ 'observed-data': '_parse_observable',
46
+ 'identity': '_load_identity'}
47
+ _stix2misp_mapping.update({galaxy_type: '_load_galaxy' for galaxy_type in _galaxy_types})
48
+ _special_mapping = {'attack-pattern': 'parse_attack_pattern',
49
+ 'course-of-action': 'parse_course_of_action',
50
+ 'vulnerability': 'parse_vulnerability'}
51
+ _timeline_mapping = {'indicator': ('valid_from', 'valid_until'),
52
+ 'observed-data': ('first_observed', 'last_observed')}
53
+
54
+ def __init__(self):
55
+ super().__init__()
56
+ self.misp_event = MISPEvent()
57
+ self.relationship = defaultdict(list)
58
+ self.tags = set()
59
+ self.galaxy = {}
60
+ self.marking_definition = {}
61
+
62
+ def handler(self, event, filename, args):
63
+ self.filename = filename
64
+ self.stix_version = f"STIX {event['spec_version'] if event.get('spec_version') else '2.1'}"
65
+ try:
66
+ event_distribution = args[0]
67
+ if not isinstance(event_distribution, int):
68
+ event_distribution = int(event_distribution) if event_distribution.isdigit() else 0
69
+ except IndexError:
70
+ event_distribution = 0
71
+ try:
72
+ attribute_distribution = args[1]
73
+ if attribute_distribution == 'event':
74
+ attribute_distribution = 5
75
+ if not isinstance(attribute_distribution, int):
76
+ attribute_distribution = int(attribute_distribution) if attribute_distribution.isdigit() else 5
77
+ except IndexError:
78
+ attribute_distribution = 5
79
+ synonyms_to_tag_names = args[2] if len(args) > 2 else '/var/www/MISP/app/files/scripts/synonymsToTagNames.json'
80
+ with open(synonyms_to_tag_names, 'rt', encoding='utf-8') as f:
81
+ self._synonyms_to_tag_names = json.loads(f.read())
82
+ self.parse_event(event)
83
+
84
+ def _load_galaxy(self, galaxy):
85
+ self.galaxy[galaxy['id'].split('--')[1]] = {'tag_names': self.parse_galaxy(galaxy), 'used': False}
86
+
87
+ def _load_identity(self, identity):
88
+ try:
89
+ self.identity[identity['id'].split('--')[1]] = identity['name']
90
+ except AttributeError:
91
+ self.identity = {identity['id'].split('--')[1]: identity['name']}
92
+
93
+ def _load_marking(self, marking):
94
+ tag = self.parse_marking(marking)
95
+ self.marking_definition[marking['id'].split('--')[1]] = {'object': tag, 'used': False}
96
+
97
+ def _load_relationship(self, relationship):
98
+ target_uuid = relationship.target_ref.split('--')[1]
99
+ reference = (target_uuid, relationship.relationship_type)
100
+ source_uuid = relationship.source_ref.split('--')[1]
101
+ self.relationship[source_uuid].append(reference)
102
+
103
+ def _load_report(self, report):
104
+ try:
105
+ self.report[report['id'].split('--')[1]] = report
106
+ except AttributeError:
107
+ self.report = {report['id'].split('--')[1]: report}
108
+
109
+ def save_file(self):
110
+ event = self.misp_event.to_json()
111
+ with open(f'{self.filename}.stix2', 'wt', encoding='utf-8') as f:
112
+ f.write(event)
113
+
114
+ ################################################################################
115
+ ## PARSING FUNCTIONS USED BY BOTH SUBCLASSES. ##
116
+ ################################################################################
117
+
118
+ def handle_markings(self):
119
+ if hasattr(self, 'marking_refs'):
120
+ for attribute in self.misp_event.attributes:
121
+ if attribute.uuid in self.marking_refs:
122
+ for marking_uuid in self.marking_refs[attribute.uuid]:
123
+ attribute.add_tag(self.marking_definition[marking_uuid]['object'])
124
+ self.marking_definition[marking_uuid]['used'] = True
125
+ if self.marking_definition:
126
+ for marking_definition in self.marking_definition.values():
127
+ if not marking_definition['used']:
128
+ self.tags.add(marking_definition['object'])
129
+ if self.tags:
130
+ for tag in self.tags:
131
+ self.misp_event.add_tag(tag)
132
+
133
+ @staticmethod
134
+ def _parse_email_body(body, references):
135
+ attributes = []
136
+ for body_multipart in body:
137
+ reference = references.pop(body_multipart['body_raw_ref'])
138
+ feature = body_multipart['content_disposition'].split(';')[0]
139
+ if feature in stix2misp_mapping.email_references_mapping:
140
+ attribute = deepcopy(stix2misp_mapping.email_references_mapping[feature])
141
+ else:
142
+ print(f'Unknown content disposition in the following email body: {body_multipart}', file=sys.stderr)
143
+ continue
144
+ if isinstance(reference, stix2.v20.observables.Artifact):
145
+ attribute.update({
146
+ 'value': body_multipart['content_disposition'].split('=')[-1].strip("'"),
147
+ 'data': reference.payload_bin,
148
+ 'to_ids': False
149
+ })
150
+ else:
151
+ attribute.update({
152
+ 'value': reference.name,
153
+ 'to_ids': False
154
+ })
155
+ attributes.append(attribute)
156
+ return attributes
157
+
158
+ @staticmethod
159
+ def _parse_email_references(email_message, references):
160
+ attributes = []
161
+ if hasattr(email_message, 'from_ref'):
162
+ reference = references.pop(email_message.from_ref)
163
+ attribute = {
164
+ 'value': reference.value,
165
+ 'to_ids': False
166
+ }
167
+ attribute.update(stix2misp_mapping.email_references_mapping['from_ref'])
168
+ attributes.append(attribute)
169
+ for feature in ('to_refs', 'cc_refs'):
170
+ if hasattr(email_message, feature):
171
+ for ref_id in getattr(email_message, feature):
172
+ reference = references.pop(ref_id)
173
+ attribute = {
174
+ 'value': reference.value,
175
+ 'to_ids': False
176
+ }
177
+ attribute.update(stix2misp_mapping.email_references_mapping[feature])
178
+ attributes.append(attribute)
179
+ return attributes
180
+
181
+ def parse_galaxies(self):
182
+ for galaxy in self.galaxy.values():
183
+ if not galaxy['used']:
184
+ for tag_name in galaxy['tag_names']:
185
+ self.tags.add(tag_name)
186
+
187
+ @staticmethod
188
+ def _parse_network_connection_reference(feature_type, feature, value):
189
+ if feature == 'type':
190
+ return {type: value.format(feature_type) for type, value in stix2misp_mapping.network_traffic_references_mapping[value].items()}
191
+ return {feature: value}
192
+
193
+ @staticmethod
194
+ def _parse_network_traffic_protocol(protocol):
195
+ return {'type': 'text', 'value': protocol, 'to_ids': False,
196
+ 'object_relation': f'layer{stix2misp_mapping.connection_protocols[protocol]}-protocol'}
197
+
198
+ @staticmethod
199
+ def _parse_observable_reference(reference, mapping, feature=None):
200
+ attribute = {
201
+ 'value': reference.value,
202
+ 'to_ids': False
203
+ }
204
+ if feature is not None:
205
+ attribute.update({key: value.format(feature) for key, value in getattr(stix2misp_mapping, mapping)[reference._type].items()})
206
+ return attribute
207
+ attribute.update({key: value for key, value in getattr(stix2misp_mapping, mapping)[reference._type].items()})
208
+ return attribute
209
+
210
+ def parse_pe(self, extension):
211
+ pe_object = MISPObject('pe', misp_objects_path_custom=_misp_objects_path)
212
+ self.fill_misp_object(pe_object, extension, 'pe_mapping')
213
+ for section in extension['sections']:
214
+ section_object = MISPObject('pe-section', misp_objects_path_custom=_misp_objects_path)
215
+ self.fill_misp_object(section_object, section, 'pe_section_mapping')
216
+ if hasattr(section, 'hashes'):
217
+ self.fill_misp_object(section_object, section.hashes, 'pe_section_mapping')
218
+ self.misp_event.add_object(section_object)
219
+ pe_object.add_reference(section_object.uuid, 'includes')
220
+ self.misp_event.add_object(pe_object)
221
+ return pe_object.uuid
222
+
223
+ def parse_relationships(self):
224
+ attribute_uuids = tuple(attribute.uuid for attribute in self.misp_event.attributes)
225
+ object_uuids = tuple(object.uuid for object in self.misp_event.objects)
226
+ for source, references in self.relationship.items():
227
+ if source in object_uuids:
228
+ source_object = self.misp_event.get_object_by_uuid(source)
229
+ for reference in references:
230
+ target, reference = reference
231
+ if target in attribute_uuids or target in object_uuids:
232
+ source_object.add_reference(target, reference)
233
+ elif source in attribute_uuids:
234
+ for attribute in self.misp_event.attributes:
235
+ if attribute.uuid == source:
236
+ for reference in references:
237
+ target, reference = reference
238
+ if target in self.galaxy:
239
+ for tag_name in self.galaxy[target]['tag_names']:
240
+ attribute.add_tag(tag_name)
241
+ self.galaxy[target]['used'] = True
242
+ break
243
+
244
+ def parse_report(self, event_uuid=None):
245
+ event_infos = set()
246
+ self.misp_event.uuid = event_uuid if event_uuid and len(self.report) > 1 else tuple(self.report.keys())[0]
247
+ for report in self.report.values():
248
+ if hasattr(report, 'name') and report.name:
249
+ event_infos.add(report.name)
250
+ if hasattr(report, 'labels') and report.labels:
251
+ for label in report.labels:
252
+ self.tags.add(label)
253
+ if hasattr(report, 'object_marking_refs') and report.object_marking_refs:
254
+ for marking_ref in report.object_marking_refs:
255
+ marking_ref = marking_ref.split('--')[1]
256
+ try:
257
+ self.tags.add(self.marking_definition[marking_ref]['object'])
258
+ self.marking_definition[marking_ref]['used'] = True
259
+ except KeyError:
260
+ continue
261
+ if hasattr(report, 'external_references'):
262
+ for reference in report.external_references:
263
+ self.misp_event.add_attribute(**{'type': 'link', 'value': reference['url']})
264
+ if len(event_infos) == 1:
265
+ self.misp_event.info = event_infos.pop()
266
+ else:
267
+ self.misp_event.info = f'Imported with MISP import script for {self.stix_version}'
268
+
269
+ @staticmethod
270
+ def _parse_user_account_groups(groups):
271
+ attributes = [{'type': 'text', 'object_relation': 'group', 'to_ids': False,
272
+ 'disable_correlation': True, 'value': group} for group in groups]
273
+ return attributes
274
+
275
+ ################################################################################
276
+ ## UTILITY FUNCTIONS. ##
277
+ ################################################################################
278
+
279
+ @staticmethod
280
+ def _choose_with_priority(container, first_choice, second_choice):
281
+ return first_choice if first_choice in container else second_choice
282
+
283
+ def filter_main_object(self, observable, main_type, test_function='_standard_test_filter'):
284
+ references = {}
285
+ main_objects = []
286
+ for key, value in observable.items():
287
+ if getattr(self, test_function)(value, main_type):
288
+ main_objects.append(value)
289
+ else:
290
+ references[key] = value
291
+ if len(main_objects) > 1:
292
+ print(f'More than one {main_type} objects in this observable: {observable}', file=sys.stderr)
293
+ return main_objects[0] if main_objects else None, references
294
+
295
+ @staticmethod
296
+ def getTimestampfromDate(date):
297
+ try:
298
+ return int(date.timestamp())
299
+ except AttributeError:
300
+ return int(time.mktime(time.strptime(date.split('+')[0], "%Y-%m-%dT%H:%M:%S.%fZ")))
301
+
302
+ @staticmethod
303
+ def _handle_data(data):
304
+ return io.BytesIO(data.encode())
305
+
306
+ @staticmethod
307
+ def parse_marking(marking):
308
+ marking_type = marking.definition_type
309
+ tag = getattr(marking.definition, marking_type)
310
+ return "{}:{}".format(marking_type, tag)
311
+
312
+ def parse_timeline(self, stix_object):
313
+ misp_object = {'timestamp': self.getTimestampfromDate(stix_object.modified)}
314
+ try:
315
+ first, last = self._timeline_mapping[stix_object._type]
316
+ first_seen = getattr(stix_object, first)
317
+ if stix_object.created != first_seen and stix_object.modified != first_seen:
318
+ misp_object['first_seen'] = first_seen
319
+ if hasattr(stix_object, last):
320
+ misp_object['last_seen'] = getattr(stix_object, last)
321
+ elif hasattr(stix_object, last):
322
+ misp_object.update({'first_seen': first_seen, 'last_seen': getattr(stix_object, last)})
323
+ except KeyError:
324
+ pass
325
+ return misp_object
326
+
327
+ @staticmethod
328
+ def _process_test_filter(value, main_type):
329
+ _is_main_process = any(feature in value for feature in ('parent_ref', 'child_refs'))
330
+ return isinstance(value, getattr(stix2.v20.observables, main_type)) and _is_main_process
331
+
332
+ @staticmethod
333
+ def _standard_test_filter(value, main_type):
334
+ return isinstance(value, getattr(stix2.v20.observables, main_type))
335
+
336
+ def update_marking_refs(self, attribute_uuid, marking_refs):
337
+ try:
338
+ self.marking_refs[attribute_uuid] = tuple(marking.split('--')[1] for marking in marking_refs)
339
+ except AttributeError:
340
+ self.marking_refs = {attribute_uuid: tuple(marking.split('--')[1] for marking in marking_refs)}
341
+
342
+
343
+ class StixFromMISPParser(StixParser):
344
+ def __init__(self):
345
+ super().__init__()
346
+ self._stix2misp_mapping.update({'custom_object': '_parse_custom'})
347
+ self._stix2misp_mapping.update({special_type: '_parse_undefined' for special_type in ('attack-pattern', 'course-of-action', 'vulnerability')})
348
+ self._custom_objects = tuple(filename.name.replace('_', '-') for filename in _misp_objects_path.glob('*') if '_' in filename.name)
349
+
350
+ def parse_event(self, stix_event):
351
+ for stix_object in stix_event.objects:
352
+ object_type = stix_object['type']
353
+ if object_type.startswith('x-misp-object'):
354
+ object_type = 'custom_object'
355
+ if object_type in self._stix2misp_mapping:
356
+ getattr(self, self._stix2misp_mapping[object_type])(stix_object)
357
+ else:
358
+ print(f'not found: {object_type}', file=sys.stderr)
359
+ if self.relationship:
360
+ self.parse_relationships()
361
+ if self.galaxy:
362
+ self.parse_galaxies()
363
+ if hasattr(self, 'report'):
364
+ self.parse_report()
365
+ self.handle_markings()
366
+
367
+ def _parse_custom(self, custom):
368
+ if 'from_object' in custom['labels']:
369
+ self.parse_custom_object(custom)
370
+ else:
371
+ self.parse_custom_attribute(custom)
372
+
373
+ def _parse_indicator(self, indicator):
374
+ if 'from_object' in indicator['labels']:
375
+ self.parse_indicator_object(indicator)
376
+ else:
377
+ self.parse_indicator_attribute(indicator)
378
+
379
+ def _parse_observable(self, observable):
380
+ if 'from_object' in observable['labels']:
381
+ self.parse_observable_object(observable)
382
+ else:
383
+ self.parse_observable_attribute(observable)
384
+
385
+ def _parse_undefined(self, stix_object):
386
+ if any(label.startswith('misp-galaxy:') for label in stix_object.get('labels', [])):
387
+ self._load_galaxy(stix_object)
388
+ else:
389
+ getattr(self, self._special_mapping[stix_object._type])(stix_object)
390
+
391
+ ################################################################################
392
+ ## PARSING FUNCTIONS. ##
393
+ ################################################################################
394
+
395
+ def fill_misp_object(self, misp_object, stix_object, mapping,
396
+ to_call='_fill_observable_object_attribute'):
397
+ for feature, value in stix_object.items():
398
+ if feature not in getattr(stix2misp_mapping, mapping):
399
+ if feature.startswith('x_misp_'):
400
+ attribute = self.parse_custom_property(feature)
401
+ if isinstance(value, list):
402
+ self._fill_misp_object_from_list(misp_object, attribute, value)
403
+ continue
404
+ else:
405
+ continue
406
+ else:
407
+ attribute = deepcopy(getattr(stix2misp_mapping, mapping)[feature])
408
+ attribute.update(getattr(self, to_call)(feature, value))
409
+ misp_object.add_attribute(**attribute)
410
+
411
+ @staticmethod
412
+ def _fill_misp_object_from_list(misp_object, mapping, values):
413
+ for value in values:
414
+ attribute = {'value': value}
415
+ attribute.update(mapping)
416
+ misp_object.add_attribute(**attribute)
417
+
418
+ def parse_attack_pattern(self, attack_pattern):
419
+ misp_object, _ = self.create_misp_object(attack_pattern)
420
+ if hasattr(attack_pattern, 'external_references'):
421
+ for reference in attack_pattern.external_references:
422
+ value = reference['external_id'].split('-')[1] if reference['source_name'] == 'capec' else reference['url']
423
+ misp_object.add_attribute(**{
424
+ 'type': 'text', 'object_relation': 'id',
425
+ 'value': value
426
+ })
427
+ self.fill_misp_object(misp_object, attack_pattern, 'attack_pattern_mapping',
428
+ '_fill_observable_object_attribute')
429
+ self.misp_event.add_object(**misp_object)
430
+
431
+ def parse_course_of_action(self, course_of_action):
432
+ misp_object, _ = self.create_misp_object(course_of_action)
433
+ self.fill_misp_object(misp_object, course_of_action, 'course_of_action_mapping',
434
+ '_fill_observable_object_attribute')
435
+ self.misp_event.add_object(**misp_object)
436
+
437
+ def parse_custom_attribute(self, custom):
438
+ attribute_type = custom['type'].split('x-misp-object-')[1]
439
+ if attribute_type not in _misp_types:
440
+ replacement = ' ' if attribute_type == 'named-pipe' else '|'
441
+ attribute_type = attribute_type.replace('-', replacement)
442
+ attribute = {'type': attribute_type,
443
+ 'timestamp': self.getTimestampfromDate(custom['modified']),
444
+ 'to_ids': bool(custom['labels'][1].split('=')[1]),
445
+ 'value': custom['x_misp_value'],
446
+ 'category': self.get_misp_category(custom['labels']),
447
+ 'uuid': custom['id'].split('--')[1]}
448
+ if custom.get('object_marking_refs'):
449
+ self.update_marking_refs(attribute['uuid'], custom['object_marking_refs'])
450
+ self.misp_event.add_attribute(**attribute)
451
+
452
+ def parse_custom_object(self, custom):
453
+ name = custom['type'].split('x-misp-object-')[1]
454
+ if name in self._custom_objects:
455
+ name = name.replace('-', '_')
456
+ misp_object = MISPObject(name, misp_objects_path_custom=_misp_objects_path)
457
+ misp_object.timestamp = self.getTimestampfromDate(custom['modified'])
458
+ misp_object.uuid = custom['id'].split('--')[1]
459
+ try:
460
+ misp_object.category = custom['category']
461
+ except KeyError:
462
+ misp_object.category = self.get_misp_category(custom['labels'])
463
+ for key, value in custom['x_misp_values'].items():
464
+ attribute_type, object_relation = key.replace('_DOT_', '.').split('_')
465
+ if isinstance(value, list):
466
+ for single_value in value:
467
+ misp_object.add_attribute(**{'type': attribute_type, 'value': single_value,
468
+ 'object_relation': object_relation})
469
+ else:
470
+ misp_object.add_attribute(**{'type': attribute_type, 'value': value,
471
+ 'object_relation': object_relation})
472
+ self.misp_event.add_object(**misp_object)
473
+
474
+ def parse_galaxy(self, galaxy):
475
+ if hasattr(galaxy, 'labels'):
476
+ return [label for label in galaxy.labels if label.startswith('misp-galaxy:')]
477
+ try:
478
+ return self._synonyms_to_tag_names[galaxy.name]
479
+ except KeyError:
480
+ print(f'Unknown {galaxy._type} name: {galaxy.name}', file=sys.stderr)
481
+ return [f'misp-galaxy:{galaxy._type}="{galaxy.name}"']
482
+
483
+ def parse_indicator_attribute(self, indicator):
484
+ attribute = self.create_attribute_dict(indicator)
485
+ attribute['to_ids'] = True
486
+ pattern = indicator.pattern.replace('\\\\', '\\')
487
+ if attribute['type'] in ('malware-sample', 'attachment'):
488
+ value, data = self.parse_attribute_pattern_with_data(pattern)
489
+ attribute.update({feature: value for feature, value in zip(('value', 'data'), (value, io.BytesIO(data.encode())))})
490
+ else:
491
+ attribute['value'] = self.parse_attribute_pattern(pattern)
492
+ self.misp_event.add_attribute(**attribute)
493
+
494
+ def parse_indicator_object(self, indicator):
495
+ misp_object, object_type = self.create_misp_object(indicator)
496
+ pattern = self._handle_pattern(indicator.pattern).replace('\\\\', '\\').split(' AND ')
497
+ try:
498
+ attributes = getattr(self, stix2misp_mapping.objects_mapping[object_type]['pattern'])(pattern)
499
+ except KeyError:
500
+ print(f"Unable to map {object_type} object:\n{indicator}", file=sys.stderr)
501
+ return
502
+ if isinstance(attributes, tuple):
503
+ attributes, target_uuid = attributes
504
+ misp_object.add_reference(target_uuid, 'includes')
505
+ for attribute in attributes:
506
+ misp_object.add_attribute(**attribute)
507
+ self.misp_event.add_object(misp_object)
508
+
509
+ def parse_observable_attribute(self, observable):
510
+ attribute = self.create_attribute_dict(observable)
511
+ attribute['to_ids'] = False
512
+ objects = observable.objects
513
+ value = self.parse_single_attribute_observable(objects, attribute['type'])
514
+ if isinstance(value, tuple):
515
+ value, data = value
516
+ attribute['data'] = data
517
+ attribute['value'] = value
518
+ self.misp_event.add_attribute(**attribute)
519
+
520
+ def parse_observable_object(self, observable):
521
+ misp_object, object_type = self.create_misp_object(observable)
522
+ observable_object = observable.objects
523
+ try:
524
+ attributes = getattr(self, stix2misp_mapping.objects_mapping[object_type]['observable'])(observable_object)
525
+ except KeyError:
526
+ print(f"Unable to map {object_type} object:\n{observable}", file=sys.stderr)
527
+ return
528
+ if isinstance(attributes, tuple):
529
+ attributes, target_uuid = attributes
530
+ misp_object.add_reference(target_uuid, 'includes')
531
+ for attribute in attributes:
532
+ misp_object.add_attribute(**attribute)
533
+ self.misp_event.add_object(misp_object)
534
+
535
+ def parse_vulnerability(self, vulnerability):
536
+ attributes = self.fill_observable_attributes(vulnerability, 'vulnerability_mapping')
537
+ if hasattr(vulnerability, 'external_references'):
538
+ for reference in vulnerability.external_references:
539
+ if reference['source_name'] == 'url':
540
+ attributes.append({'type': 'link', 'object_relation': 'references', 'value': reference['url']})
541
+ if len(attributes) > 1:
542
+ vulnerability_object, _ = self.create_misp_object(vulnerability)
543
+ for attribute in attributes:
544
+ vulnerability_object.add_attribute(**attribute)
545
+ self.misp_event.add_object(**vulnerability_object)
546
+ else:
547
+ attribute = self.create_attribute_dict(vulnerability)
548
+ attribute['value'] = attributes[0]['value']
549
+ self.misp_event.add_attribute(**attribute)
550
+
551
+ ################################################################################
552
+ ## OBSERVABLE PARSING FUNCTIONS ##
553
+ ################################################################################
554
+
555
+ @staticmethod
556
+ def _define_hash_type(hash_type):
557
+ if 'sha' in hash_type:
558
+ return f'SHA-{hash_type.split("sha")[1]}'
559
+ return hash_type.upper() if hash_type == 'md5' else hash_type
560
+
561
+ @staticmethod
562
+ def _fetch_file_observable(observable_objects):
563
+ for key, observable in observable_objects.items():
564
+ if observable['type'] == 'file':
565
+ return key
566
+ return '0'
567
+
568
+ @staticmethod
569
+ def _fill_observable_attribute(attribute_type, object_relation, value):
570
+ return {'type': attribute_type,
571
+ 'object_relation': object_relation,
572
+ 'value': value,
573
+ 'to_ids': False}
574
+
575
+ def fill_observable_attributes(self, observable, object_mapping):
576
+ attributes = []
577
+ for key, value in observable.items():
578
+ if key in getattr(stix2misp_mapping, object_mapping):
579
+ attribute = deepcopy(getattr(stix2misp_mapping, object_mapping)[key])
580
+ elif key.startswith('x_misp_'):
581
+ attribute = self.parse_custom_property(key)
582
+ if isinstance(value, list):
583
+ for single_value in value:
584
+ single_attribute = {'value': single_value, 'to_ids': False}
585
+ single_attribute.update(attribute)
586
+ attributes.append(single_attribute)
587
+ continue
588
+ else:
589
+ continue
590
+ attribute.update({'value': value, 'to_ids': False})
591
+ attributes.append(attribute)
592
+ return attributes
593
+
594
+ def _handle_multiple_file_fields(self, file):
595
+ attributes = []
596
+ for feature, attribute_type in zip(('filename', 'path', 'fullpath'), ('filename', 'text', 'text')):
597
+ key = f'x_misp_multiple_{feature}'
598
+ if key in file:
599
+ attributes.append(self._fill_observable_attribute(attribute_type, feature, file.pop(key)))
600
+ elif f'{key}s' in file:
601
+ attributes.extend(self._fill_observable_attribute(attribute_type, feature, value) for value in file.pop(key))
602
+ attributes.extend(self.fill_observable_attributes(file, 'file_mapping'))
603
+ return attributes
604
+
605
+ def parse_asn_observable(self, observable):
606
+ attributes = []
607
+ mapping = 'asn_mapping'
608
+ for observable_object in observable.values():
609
+ if isinstance(observable_object, stix2.v20.observables.AutonomousSystem):
610
+ attributes.extend(self.fill_observable_attributes(observable_object, mapping))
611
+ else:
612
+ attributes.append(self._parse_observable_reference(observable_object, mapping))
613
+ return attributes
614
+
615
+ def _parse_attachment(self, observable):
616
+ if len(observable) > 1:
617
+ return self._parse_name(observable, index='1'), self._parse_payload(observable)
618
+ return self._parse_name(observable)
619
+
620
+ def parse_credential_observable(self, observable):
621
+ return self.fill_observable_attributes(observable['0'], 'credential_mapping')
622
+
623
+ def _parse_domain_ip_attribute(self, observable):
624
+ return f'{self._parse_value(observable)}|{self._parse_value(observable, index="1")}'
625
+
626
+ @staticmethod
627
+ def parse_domain_ip_observable(observable):
628
+ attributes = []
629
+ for observable_object in observable.values():
630
+ attribute = deepcopy(stix2misp_mapping.domain_ip_mapping[observable_object._type])
631
+ attribute.update({'value': observable_object.value, 'to_ids': False})
632
+ attributes.append(attribute)
633
+ return attributes
634
+
635
+ @staticmethod
636
+ def _parse_email_message(observable, attribute_type):
637
+ return observable['0'].get(attribute_type.split('-')[1])
638
+
639
+ def parse_email_observable(self, observable):
640
+ email, references = self.filter_main_object(observable, 'EmailMessage')
641
+ attributes = self.fill_observable_attributes(email, 'email_mapping')
642
+ if hasattr(email, 'additional_header_fields'):
643
+ attributes.extend(self.fill_observable_attributes(email.additional_header_fields, 'email_mapping'))
644
+ attributes.extend(self._parse_email_references(email, references))
645
+ if hasattr(email, 'body_multipart') and email.body_multipart:
646
+ attributes.extend(self._parse_email_body(email.body_multipart, references))
647
+ return attributes
648
+
649
+ @staticmethod
650
+ def _parse_email_reply_to(observable):
651
+ return observable['0'].additional_header_fields.get('Reply-To')
652
+
653
+ def parse_file_observable(self, observable):
654
+ file, references = self.filter_main_object(observable, 'File')
655
+ references = {key: {'object': value, 'used': False} for key, value in references.items()}
656
+ file = {key: value for key, value in file.items()}
657
+ multiple_fields = any(f'x_misp_multiple_{feature}' in file for feature in ('filename', 'path', 'fullpath'))
658
+ attributes = self._handle_multiple_file_fields(file) if multiple_fields else self.fill_observable_attributes(file, 'file_mapping')
659
+ if 'hashes' in file:
660
+ attributes.extend(self.fill_observable_attributes(file['hashes'], 'file_mapping'))
661
+ if 'content_ref' in file:
662
+ reference = references[file['content_ref']]
663
+ value = f'{reference["object"].name}|{reference["object"].hashes["MD5"]}'
664
+ attributes.append({'type': 'malware-sample', 'object_relation': 'malware-sample', 'value': value,
665
+ 'to_ids': False, 'data': reference['object'].payload_bin})
666
+ reference['used'] = True
667
+ if 'parent_directory_ref' in file:
668
+ reference = references[file['parent_directory_ref']]
669
+ attributes.append({'type': 'text', 'object_relation': 'path',
670
+ 'value': reference['object'].path, 'to_ids': False})
671
+ reference['used'] = True
672
+ for reference in references.values():
673
+ if not reference['used']:
674
+ attributes.append({
675
+ 'type': 'attachment',
676
+ 'object_relation': 'attachment',
677
+ 'value': reference['object'].name,
678
+ 'data': reference['object'].payload_bin,
679
+ 'to_ids': False
680
+ })
681
+ return attributes
682
+
683
+ def _parse_filename_hash(self, observable, attribute_type, index='0'):
684
+ hash_type = attribute_type.split('|')[1]
685
+ filename = self._parse_name(observable, index=index)
686
+ hash_value = self._parse_hash(observable, hash_type, index=index)
687
+ return f'{filename}|{hash_value}'
688
+
689
+ def _parse_hash(self, observable, attribute_type, index='0'):
690
+ hash_type = self._define_hash_type(attribute_type)
691
+ return observable[index]['hashes'].get(hash_type)
692
+
693
+ def parse_ip_port_observable(self, observable):
694
+ network_traffic, references = self.filter_main_object(observable, 'NetworkTraffic')
695
+ attributes = []
696
+ for feature in ('src', 'dst'):
697
+ port = f'{feature}_port'
698
+ if hasattr(network_traffic, port):
699
+ attribute = deepcopy(stix2misp_mapping.ip_port_mapping[port])
700
+ attribute.update({'value': getattr(network_traffic, port), 'to_ids': False})
701
+ attributes.append(attribute)
702
+ ref = f'{feature}_ref'
703
+ if hasattr(network_traffic, ref):
704
+ attributes.append(self._parse_observable_reference(references.pop(getattr(network_traffic, ref)), 'ip_port_references_mapping', feature))
705
+ for reference in references.values():
706
+ attribute = deepcopy(stix2misp_mapping.ip_port_references_mapping[reference._type])
707
+ attribute.update({'value': reference.value, 'to_ids': False})
708
+ attributes.append(attribute)
709
+ return attributes
710
+
711
+ def _parse_malware_sample(self, observable):
712
+ if len(observable) > 1:
713
+ value = self._parse_filename_hash(observable, 'filename|md5', '1')
714
+ return value, self._parse_payload(observable)
715
+ return self._parse_filename_hash(observable, 'filename|md5')
716
+
717
+ @staticmethod
718
+ def _parse_name(observable, index='0'):
719
+ return observable[index].get('name')
720
+
721
+ def _parse_network_attribute(self, observable):
722
+ port = self._parse_port(observable, index='1')
723
+ return f'{self._parse_value(observable)}|{port}'
724
+
725
+ def parse_network_connection_observable(self, observable):
726
+ network_traffic, references = self.filter_main_object(observable, 'NetworkTraffic')
727
+ attributes = self._parse_network_traffic(network_traffic, references)
728
+ if hasattr(network_traffic, 'protocols'):
729
+ attributes.extend(self._parse_network_traffic_protocol(protocol) for protocol in network_traffic.protocols if protocol in stix2misp_mapping.connection_protocols)
730
+ if references:
731
+ for reference in references.values():
732
+ attributes.append(self._parse_observable_reference(reference, 'domain_ip_mapping'))
733
+ return attributes
734
+
735
+ def parse_network_socket_observable(self, observable):
736
+ network_traffic, references = self.filter_main_object(observable, 'NetworkTraffic')
737
+ attributes = self._parse_network_traffic(network_traffic, references)
738
+ if hasattr(network_traffic, 'protocols'):
739
+ attributes.append({'type': 'text', 'object_relation': 'protocol', 'to_ids': False,
740
+ 'value': network_traffic.protocols[0].strip("'")})
741
+ if hasattr(network_traffic, 'extensions') and network_traffic.extensions:
742
+ attributes.extend(self._parse_socket_extension(network_traffic.extensions['socket-ext']))
743
+ if references:
744
+ for reference in references.values():
745
+ attributes.append(self._parse_observable_reference(reference, 'domain_ip_mapping'))
746
+ return attributes
747
+
748
+ def _parse_network_traffic(self, network_traffic, references):
749
+ attributes = []
750
+ mapping = 'network_traffic_references_mapping'
751
+ for feature in ('src', 'dst'):
752
+ port = f'{feature}_port'
753
+ if hasattr(network_traffic, port):
754
+ attribute = deepcopy(stix2misp_mapping.network_traffic_mapping[port])
755
+ attribute.update({'value': getattr(network_traffic, port), 'to_ids': False})
756
+ attributes.append(attribute)
757
+ ref = f'{feature}_ref'
758
+ if hasattr(network_traffic, ref):
759
+ attributes.append(self._parse_observable_reference(references.pop(getattr(network_traffic, ref)), mapping, feature))
760
+ if hasattr(network_traffic, f'{ref}s'):
761
+ for ref in getattr(network_traffic, f'{ref}s'):
762
+ attributes.append(self._parse_observable_reference(references.pop(ref), mapping, feature))
763
+ return attributes
764
+
765
+ @staticmethod
766
+ def _parse_number(observable):
767
+ return observable['0'].get('number')
768
+
769
+ @staticmethod
770
+ def _parse_payload(observable):
771
+ return observable['0'].payload_bin
772
+
773
+ def parse_pe_observable(self, observable):
774
+ key = self._fetch_file_observable(observable)
775
+ extension = observable[key]['extensions']['windows-pebinary-ext']
776
+ pe_uuid = self.parse_pe(extension)
777
+ return self.parse_file_observable(observable), pe_uuid
778
+
779
+ @staticmethod
780
+ def _parse_port(observable, index='0'):
781
+ port_observable = observable[index]
782
+ return port_observable['src_port'] if 'src_port' in port_observable else port_observable['dst_port']
783
+
784
+ def parse_process_observable(self, observable):
785
+ process, references = self.filter_main_object(observable, 'Process', test_function='_process_test_filter')
786
+ attributes = self.fill_observable_attributes(process, 'process_mapping')
787
+ if hasattr(process, 'parent_ref'):
788
+ attributes.extend(self.fill_observable_attributes(references[process.parent_ref], 'parent_process_reference_mapping'))
789
+ if hasattr(process, 'child_refs'):
790
+ for reference in process.child_refs:
791
+ attributes.extend(self.fill_observable_attributes(references[reference], 'child_process_reference_mapping'))
792
+ if hasattr(process, 'binary_ref'):
793
+ reference = references[process.binary_ref]
794
+ attribute = deepcopy(stix2misp_mapping.process_image_mapping)
795
+ attribute.update({'value': reference.name, 'to_ids': False})
796
+ attributes.append(attribute)
797
+ return attributes
798
+
799
+ @staticmethod
800
+ def _parse_regkey_attribute(observable):
801
+ return observable['0'].get('key')
802
+
803
+ def parse_regkey_observable(self, observable):
804
+ attributes = []
805
+ for key, value in observable['0'].items():
806
+ if key in stix2misp_mapping.regkey_mapping:
807
+ attribute = deepcopy(stix2misp_mapping.regkey_mapping[key])
808
+ attribute.update({'value': value.replace('\\\\', '\\'), 'to_ids': False})
809
+ attributes.append(attribute)
810
+ if 'values' in observable['0']:
811
+ attributes.extend(self.fill_observable_attributes(observable['0']['values'][0], 'regkey_mapping'))
812
+ return attributes
813
+
814
+ def _parse_regkey_value(self, observable):
815
+ regkey = self._parse_regkey_attribute(observable)
816
+ return f'{regkey}|{observable["0"]["values"][0].get("data")}'
817
+
818
+ def parse_single_attribute_observable(self, observable, attribute_type):
819
+ if attribute_type in stix2misp_mapping.attributes_type_mapping:
820
+ return getattr(self, stix2misp_mapping.attributes_type_mapping[attribute_type])(observable, attribute_type)
821
+ return getattr(self, stix2misp_mapping.attributes_mapping[attribute_type])(observable)
822
+
823
+ def _parse_socket_extension(self, extension):
824
+ attributes = []
825
+ extension = {key: value for key, value in extension.items()}
826
+ if 'x_misp_text_address_family' in extension:
827
+ extension.pop('address_family')
828
+ for element, value in extension.items():
829
+ if element in stix2misp_mapping.network_socket_extension_mapping:
830
+ attribute = deepcopy(stix2misp_mapping.network_socket_extension_mapping[element])
831
+ if element in ('is_listening', 'is_blocking'):
832
+ if value is False:
833
+ continue
834
+ value = element.split('_')[1]
835
+ elif element.startswith('x_misp_'):
836
+ attribute = self.parse_custom_property(element)
837
+ else:
838
+ continue
839
+ attribute.update({'value': value, 'to_ids': False})
840
+ attributes.append(attribute)
841
+ return attributes
842
+
843
+ @staticmethod
844
+ def parse_url_observable(observable):
845
+ attributes = []
846
+ for object in observable.values():
847
+ feature = 'dst_port' if isinstance(object, stix2.v20.observables.NetworkTraffic) else 'value'
848
+ attribute = deepcopy(stix2misp_mapping.url_mapping[object._type])
849
+ attribute.update({'value': getattr(object, feature), 'to_ids': False})
850
+ attributes.append(attribute)
851
+ return attributes
852
+
853
+ def parse_user_account_observable(self, observable):
854
+ observable = observable['0']
855
+ attributes = self.fill_observable_attributes(observable, 'user_account_mapping')
856
+ if 'extensions' in observable and 'unix-account-ext' in observable['extensions']:
857
+ extension = observable['extensions']['unix-account-ext']
858
+ if 'groups' in extension:
859
+ attributes.extend(self._parse_user_account_groups(extension['groups']))
860
+ attributes.extend(self.fill_observable_attributes(extension, 'user_account_mapping'))
861
+ return attributes
862
+
863
+ @staticmethod
864
+ def _parse_value(observable, index='0'):
865
+ return observable[index].get('value')
866
+
867
+ def _parse_x509_attribute(self, observable, attribute_type):
868
+ hash_type = attribute_type.split('-')[-1]
869
+ return self._parse_hash(observable, hash_type)
870
+
871
+ def parse_x509_observable(self, observable):
872
+ attributes = self.fill_observable_attributes(observable['0'], 'x509_mapping')
873
+ if hasattr(observable['0'], 'hashes') and observable['0'].hashes:
874
+ attributes.extend(self.fill_observable_attributes(observable['0'].hashes, 'x509_mapping'))
875
+ return attributes
876
+
877
+ ################################################################################
878
+ ## PATTERN PARSING FUNCTIONS. ##
879
+ ################################################################################
880
+
881
+ def fill_pattern_attributes(self, pattern, object_mapping):
882
+ attributes = []
883
+ for pattern_part in pattern:
884
+ pattern_type, pattern_value = pattern_part.split(' = ')
885
+ if pattern_type not in getattr(stix2misp_mapping, object_mapping):
886
+ if 'x_misp_' in pattern_type:
887
+ attribute = self.parse_custom_property(pattern_type)
888
+ attribute['value'] = pattern_value.strip("'")
889
+ attributes.append(attribute)
890
+ continue
891
+ attribute = deepcopy(getattr(stix2misp_mapping, object_mapping)[pattern_type])
892
+ attribute['value'] = pattern_value.strip("'")
893
+ attributes.append(attribute)
894
+ return attributes
895
+
896
+ def parse_asn_pattern(self, pattern):
897
+ return self.fill_pattern_attributes(pattern, 'asn_mapping')
898
+
899
+ def parse_credential_pattern(self, pattern):
900
+ return self.fill_pattern_attributes(pattern, 'credential_mapping')
901
+
902
+ def parse_domain_ip_pattern(self, pattern):
903
+ return self.fill_pattern_attributes(pattern, 'domain_ip_mapping')
904
+
905
+ def parse_email_pattern(self, pattern):
906
+ attributes = []
907
+ attachments = defaultdict(dict)
908
+ for pattern_part in pattern:
909
+ pattern_type, pattern_value = pattern_part.split(' = ')
910
+ if 'body_multipart' in pattern_type:
911
+ pattern_type = pattern_type.split('.')
912
+ feature = 'data' if pattern_type[-1] == 'payload_bin' else 'value'
913
+ attachments[pattern_type[0][-2]][feature] = pattern_value.strip("'")
914
+ continue
915
+ if pattern_type not in stix2misp_mapping.email_mapping:
916
+ if 'x_misp_' in pattern_type:
917
+ attribute = self.parse_custom_property(pattern_type)
918
+ attribute['value'] = pattern_value.strip("'")
919
+ attributes.append(attribute)
920
+ continue
921
+ attribute = deepcopy(stix2misp_mapping.email_mapping[pattern_type])
922
+ attribute['value'] = pattern_value.strip("'")
923
+ attributes.append(attribute)
924
+ for attachment in attachments.values():
925
+ if 'data' in attachment:
926
+ attribute = {'type': 'attachment', 'object_relation': 'screenshot', 'data': attachment['data']}
927
+ else:
928
+ attribute = {'type': 'email-attachment', 'object_relation': 'attachment'}
929
+ attribute['value'] = attachment['value']
930
+ attributes.append(attribute)
931
+ return attributes
932
+
933
+ def parse_file_pattern(self, pattern):
934
+ attributes = []
935
+ attachment = {}
936
+ for pattern_part in pattern:
937
+ pattern_type, pattern_value = pattern_part.split(' = ')
938
+ if pattern_type in stix2misp_mapping.attachment_types:
939
+ attachment[pattern_type] = pattern_value.strip("'")
940
+ if pattern_type not in stix2misp_mapping.file_mapping:
941
+ continue
942
+ attribute = deepcopy(stix2misp_mapping.file_mapping[pattern_type])
943
+ attribute['value'] = pattern_value.strip("'")
944
+ attributes.append(attribute)
945
+ if 'file:content_ref.payload_bin' in attachment:
946
+ filename = self._choose_with_priority(attachment, 'file:content_ref.name', 'file:name')
947
+ md5 = self._choose_with_priority(attachment, "file:content_ref.hashes.'MD5'", "file:hashes.'MD5'")
948
+ attributes.append({
949
+ 'type': 'malware-sample',
950
+ 'object_relation': 'malware-sample',
951
+ 'value': f'{attachment[filename]}|{attachment[md5]}',
952
+ 'data': attachment['file:content_ref.payload_bin']
953
+ })
954
+ if 'artifact:payload_bin' in attachment:
955
+ attributes.append({
956
+ 'type': 'attachment',
957
+ 'object_relation': 'attachment',
958
+ 'value': attachment['artifact:x_misp_text_name'] if 'artifact:x_misp_text_name' in attachment else attachment['file:name'],
959
+ 'data': attachment['artifact:payload_bin']
960
+ })
961
+ return attributes
962
+
963
+ def parse_ip_port_pattern(self, pattern):
964
+ return self.fill_pattern_attributes(pattern, 'ip_port_mapping')
965
+
966
+ def parse_network_connection_pattern(self, pattern):
967
+ attributes = []
968
+ references = defaultdict(dict)
969
+ for pattern_part in pattern:
970
+ pattern_type, pattern_value = pattern_part.split(' = ')
971
+ if pattern_type not in stix2misp_mapping.network_traffic_mapping:
972
+ pattern_value = pattern_value.strip("'")
973
+ if pattern_type.startswith('network-traffic:protocols['):
974
+ attributes.append({
975
+ 'type': 'text', 'value': pattern_value,
976
+ 'object_relation': f'layer{stix2misp_mapping.connection_protocols[pattern_value]}-protocol'
977
+ })
978
+ elif any(pattern_type.startswith(f'network-traffic:{feature}_ref') for feature in ('src', 'dst')):
979
+ feature_type, ref = pattern_type.split(':')[1].split('_')
980
+ ref, feature = ref.split('.')
981
+ ref = f"{feature_type}_{'0' if ref == 'ref' else ref.strip('ref[]')}"
982
+ references[ref].update(self._parse_network_connection_reference(feature_type, feature, pattern_value))
983
+ continue
984
+ attribute = deepcopy(stix2misp_mapping.network_traffic_mapping[pattern_type])
985
+ attribute['value'] = pattern_value.strip("'")
986
+ attributes.append(attribute)
987
+ attributes.extend(attribute for attribute in references.values())
988
+ return attributes
989
+
990
+ def parse_network_socket_pattern(self, pattern):
991
+ attributes = []
992
+ references = defaultdict(dict)
993
+ for pattern_part in pattern:
994
+ pattern_type, pattern_value = pattern_part.split(' = ')
995
+ pattern_value = pattern_value.strip("'")
996
+ if pattern_type not in stix2misp_mapping.network_traffic_mapping:
997
+ if pattern_type in stix2misp_mapping.network_socket_extension_mapping:
998
+ attribute = deepcopy(stix2misp_mapping.network_socket_extension_mapping[pattern_type])
999
+ if pattern_type.startswith("network-traffic:extensions.'socket-ext'.is_"):
1000
+ if pattern_value != 'True':
1001
+ continue
1002
+ pattern_value = pattern_type.split('_')[1]
1003
+ else:
1004
+ if pattern_type.startswith('network-traffic:protocols['):
1005
+ attributes.append({'type': 'text', 'object_relation': 'protocol', 'value': pattern_value})
1006
+ elif any(pattern_type.startswith(f'network-traffic:{feature}_ref') for feature in ('src', 'dst')):
1007
+ feature_type, ref = pattern_type.split(':')[1].split('_')
1008
+ ref, feature = ref.split('.')
1009
+ ref = f"{feature_type}_{'0' if ref == 'ref' else ref.strip('ref[]')}"
1010
+ references[ref].update(self._parse_network_connection_reference(feature_type, feature, pattern_value))
1011
+ continue
1012
+ else:
1013
+ attribute = deepcopy(stix2misp_mapping.network_traffic_mapping[pattern_type])
1014
+ attribute['value'] = pattern_value
1015
+ attributes.append(attribute)
1016
+ attributes.extend(attribute for attribute in references.values())
1017
+ return attributes
1018
+
1019
+ def parse_pe_pattern(self, pattern):
1020
+ attributes = []
1021
+ sections = defaultdict(dict)
1022
+ pe = MISPObject('pe', misp_objects_path_custom=_misp_objects_path)
1023
+ for pattern_part in pattern:
1024
+ pattern_type, pattern_value = pattern_part.split(' = ')
1025
+ if ':extensions.' in pattern_type:
1026
+ if '.sections[' in pattern_type:
1027
+ pattern_type = pattern_type.split('.')
1028
+ relation = pattern_type[-1].strip("'")
1029
+ if relation in stix2misp_mapping.pe_section_mapping:
1030
+ sections[pattern_type[2][-2]][relation] = pattern_value.strip("'")
1031
+ else:
1032
+ pattern_type = pattern_type.split('.')[-1]
1033
+ if pattern_type not in stix2misp_mapping.pe_mapping:
1034
+ if pattern_type.startswith('x_misp_'):
1035
+ attribute = self.parse_custom_property(pattern_type)
1036
+ attribute['value'] = pattern_value.strip("'")
1037
+ pe.add_attribute(**attribute)
1038
+ continue
1039
+ attribute = deepcopy(stix2misp_mapping.pe_mapping[pattern_type])
1040
+ attribute['value'] = pattern_value.strip("'")
1041
+ pe.add_attribute(**attribute)
1042
+ else:
1043
+ if pattern_type not in stix2misp_mapping.file_mapping:
1044
+ if pattern_type.startswith('x_misp_'):
1045
+ attribute = self.parse_custom_property(pattern_type)
1046
+ attribute['value'] = pattern_value.strip("'")
1047
+ attributes.append(attribute)
1048
+ continue
1049
+ attribute = deepcopy(stix2misp_mapping.file_mapping[pattern_type])
1050
+ attribute['value'] = pattern_value.strip("'")
1051
+ attributes.append(attribute)
1052
+ for section in sections.values():
1053
+ pe_section = MISPObject('pe-section', misp_objects_path_custom=_misp_objects_path)
1054
+ for feature, value in section.items():
1055
+ attribute = deepcopy(stix2misp_mapping.pe_section_mapping[feature])
1056
+ attribute['value'] = value
1057
+ pe_section.add_attribute(**attribute)
1058
+ self.misp_event.add_object(pe_section)
1059
+ pe.add_reference(pe_section.uuid, 'includes')
1060
+ self.misp_event.add_object(pe)
1061
+ return attributes, pe.uuid
1062
+
1063
+ def parse_process_pattern(self, pattern):
1064
+ return self.fill_pattern_attributes(pattern, 'process_mapping')
1065
+
1066
+ def parse_regkey_pattern(self, pattern):
1067
+ return self.fill_pattern_attributes(pattern, 'regkey_mapping')
1068
+
1069
+ def parse_url_pattern(self, pattern):
1070
+ return self.fill_pattern_attributes(pattern, 'url_mapping')
1071
+
1072
+ @staticmethod
1073
+ def parse_user_account_pattern(pattern):
1074
+ attributes = []
1075
+ for pattern_part in pattern:
1076
+ pattern_type, pattern_value = pattern_part.split(' = ')
1077
+ pattern_type = pattern_type.split('.')[-1].split('[')[0] if "extensions.'unix-account-ext'" in pattern_type else pattern_type.split(':')[-1]
1078
+ if pattern_type not in stix2misp_mapping.user_account_mapping:
1079
+ if pattern_type.startswith('group'):
1080
+ attributes.append({'type': 'text', 'object_relation': 'group', 'value': pattern_value.strip("'")})
1081
+ continue
1082
+ attribute = deepcopy(stix2misp_mapping.user_account_mapping[pattern_type])
1083
+ attribute['value'] = pattern_value.strip("'")
1084
+ attributes.append(attribute)
1085
+ return attributes
1086
+
1087
+ def parse_x509_pattern(self, pattern):
1088
+ return self.fill_pattern_attributes(pattern, 'x509_mapping')
1089
+
1090
+ ################################################################################
1091
+ ## UTILITY FUNCTIONS. ##
1092
+ ################################################################################
1093
+
1094
+ def create_attribute_dict(self, stix_object):
1095
+ labels = stix_object['labels']
1096
+ attribute_uuid = stix_object.id.split('--')[1]
1097
+ attribute = {'uuid': attribute_uuid,
1098
+ 'type': self.get_misp_type(labels),
1099
+ 'category': self.get_misp_category(labels)}
1100
+ tags = [{'name': label} for label in labels[3:]]
1101
+ if tags:
1102
+ attribute['Tag'] = tags
1103
+ attribute.update(self.parse_timeline(stix_object))
1104
+ if hasattr(stix_object, 'description') and stix_object.description:
1105
+ attribute['comment'] = stix_object.description
1106
+ if hasattr(stix_object, 'object_marking_refs'):
1107
+ self.update_marking_refs(attribute_uuid, stix_object.object_marking_refs)
1108
+ return attribute
1109
+
1110
+ def create_misp_object(self, stix_object):
1111
+ labels = stix_object['labels']
1112
+ object_type = self.get_misp_type(labels)
1113
+ misp_object = MISPObject('file' if object_type == 'WindowsPEBinaryFile' else object_type,
1114
+ misp_objects_path_custom=_misp_objects_path)
1115
+ misp_object.uuid = stix_object.id.split('--')[1]
1116
+ if hasattr(stix_object, 'description') and stix_object.description:
1117
+ misp_object.comment = stix_object.description
1118
+ misp_object.update(self.parse_timeline(stix_object))
1119
+ return misp_object, object_type
1120
+
1121
+ @staticmethod
1122
+ def _fill_object_attribute(feature, value):
1123
+ return {'value': str(value) if feature in ('entropy', 'size') else value}
1124
+
1125
+ @staticmethod
1126
+ def _fill_observable_object_attribute(feature, value):
1127
+ return {'value': str(value) if feature in ('entropy', 'size') else value,
1128
+ 'to_ids': False}
1129
+
1130
+ @staticmethod
1131
+ def get_misp_category(labels):
1132
+ return labels[1].split('=')[1].strip('"')
1133
+
1134
+ @staticmethod
1135
+ def get_misp_type(labels):
1136
+ return labels[0].split('=')[1].strip('"')
1137
+
1138
+ @staticmethod
1139
+ def parse_attribute_pattern(pattern):
1140
+ if ' AND ' in pattern:
1141
+ pattern_parts = pattern.strip('[]').split(' AND ')
1142
+ if len(pattern_parts) == 3:
1143
+ _, value1 = pattern_parts[2].split(' = ')
1144
+ _, value2 = pattern_parts[0].split(' = ')
1145
+ return '{}|{}'.format(value1.strip("'"), value2.strip("'"))
1146
+ else:
1147
+ _, value1 = pattern_parts[0].split(' = ')
1148
+ _, value2 = pattern_parts[1].split(' = ')
1149
+ if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
1150
+ return value2.strip("'")
1151
+ return '{}|{}'.format(value1.strip("'"), value2.strip("'"))
1152
+ else:
1153
+ return pattern.split(' = ')[1].strip("]'")
1154
+
1155
+ def parse_attribute_pattern_with_data(self, pattern):
1156
+ if 'file:content_ref.payload_bin' not in pattern:
1157
+ return self.parse_attribute_pattern(pattern)
1158
+ pattern_parts = pattern.strip('[]').split(' AND ')
1159
+ if len(pattern_parts) == 3:
1160
+ filename = pattern_parts[0].split(' = ')[1]
1161
+ md5 = pattern_parts[1].split(' = ')[1]
1162
+ return "{}|{}".format(filename.strip("'"), md5.strip("'")), pattern_parts[2].split(' = ')[1].strip("'")
1163
+ return pattern_parts[0].split(' = ')[1].strip("'"), pattern_parts[1].split(' = ')[1].strip("'")
1164
+
1165
+ @staticmethod
1166
+ def parse_custom_property(custom_property):
1167
+ properties = custom_property.split('_')
1168
+ return {'type': properties[2], 'object_relation': '-'.join(properties[3:])}
1169
+
1170
+
1171
+ class ExternalStixParser(StixParser):
1172
+ def __init__(self):
1173
+ super().__init__()
1174
+ self._stix2misp_mapping.update({'attack-pattern': 'parse_attack_pattern',
1175
+ 'course-of-action': 'parse_course_of_action',
1176
+ 'vulnerability': 'parse_vulnerability'})
1177
+
1178
+ ################################################################################
1179
+ ## PARSING FUNCTIONS. ##
1180
+ ################################################################################
1181
+
1182
+ def parse_event(self, stix_event):
1183
+ for stix_object in stix_event.objects:
1184
+ object_type = stix_object['type']
1185
+ if object_type in self._stix2misp_mapping:
1186
+ getattr(self, self._stix2misp_mapping[object_type])(stix_object)
1187
+ else:
1188
+ print(f'not found: {object_type}', file=sys.stderr)
1189
+ if self.relationship:
1190
+ self.parse_relationships()
1191
+ if self.galaxy:
1192
+ self.parse_galaxies()
1193
+ event_uuid = stix_event.id.split('--')[1]
1194
+ if hasattr(self, 'report'):
1195
+ self.parse_report(event_uuid=event_uuid)
1196
+ else:
1197
+ self.misp_event.uuid = event_uuid
1198
+ self.misp_event.info = 'Imported with the STIX to MISP import script.'
1199
+ self.handle_markings()
1200
+
1201
+ def parse_galaxy(self, galaxy):
1202
+ galaxy_names = self._check_existing_galaxy_name(galaxy.name)
1203
+ if galaxy_names is not None:
1204
+ return galaxy_names
1205
+ return [f'misp-galaxy:{galaxy._type}="{galaxy.name}"']
1206
+
1207
+ def _parse_indicator(self, indicator):
1208
+ pattern = indicator.pattern
1209
+ if any(relation in pattern for relation in stix2misp_mapping.pattern_forbidden_relations) or all(relation in pattern for relation in (' OR ', ' AND ')):
1210
+ self.add_stix2_pattern_object(indicator)
1211
+ separator = ' OR ' if ' OR ' in pattern else ' AND '
1212
+ self.parse_usual_indicator(indicator, separator)
1213
+
1214
+ def _parse_observable(self, observable):
1215
+ types = self._parse_observable_types(observable.objects)
1216
+ try:
1217
+ getattr(self, stix2misp_mapping.observable_mapping[types])(observable)
1218
+ except KeyError:
1219
+ print(f'Type(s) not supported at the moment: {types}\n', file=sys.stderr)
1220
+
1221
+ def _parse_undefined(self, stix_object):
1222
+ try:
1223
+ self.objects_to_parse[stix_object['id'].split('--')[1]] = stix_object
1224
+ except AttributeError:
1225
+ self.objects_to_parse = {stix_object['id'].split('--')[1]: stix_object}
1226
+
1227
+ def add_stix2_pattern_object(self, indicator):
1228
+ misp_object = MISPObject('stix2-pattern', misp_objects_path_custom=_misp_objects_path)
1229
+ misp_object.uuid = indicator.id.split('--')[1]
1230
+ misp_object.update(self.parse_timeline(indicator))
1231
+ version = f'STIX {indicator.pattern_version}' if hasattr(indicator, 'pattern_version') else 'STIX 2.0'
1232
+ misp_object.add_attribute(**{'type': 'text', 'object_relation': 'version', 'value': version})
1233
+ misp_object.add_attribute(**{'type': 'stix2-pattern', 'object_relation': 'stix2-pattern',
1234
+ 'value': indicator.pattern})
1235
+ self.misp_event.add_object(**misp_object)
1236
+
1237
+ @staticmethod
1238
+ def fill_misp_object(misp_object, stix_object, mapping):
1239
+ for key, feature in getattr(stix2misp_mapping, mapping).items():
1240
+ if hasattr(stix_object, key):
1241
+ attribute = deepcopy(feature)
1242
+ attribute['value'] = getattr(stix_object, key)
1243
+ misp_object.add_attribute(**attribute)
1244
+
1245
+ @staticmethod
1246
+ def fill_misp_object_from_dict(misp_object, stix_object, mapping):
1247
+ for key, feature in getattr(stix2misp_mapping, mapping).items():
1248
+ if key in stix_object:
1249
+ attribute = deepcopy(feature)
1250
+ attribute['value'] = stix_object[key]
1251
+ misp_object.add_attribute(**attribute)
1252
+
1253
+ def parse_attack_pattern(self, attack_pattern):
1254
+ galaxy_names = self._check_existing_galaxy_name(attack_pattern.name)
1255
+ if galaxy_names is not None:
1256
+ self.galaxy[attack_pattern['id'].split('--')[1]] = {'tag_names': galaxy_names, 'used': False}
1257
+ else:
1258
+ misp_object = self.create_misp_object(attack_pattern)
1259
+ if hasattr(attack_pattern, 'external_references'):
1260
+ for reference in attack_pattern.external_references:
1261
+ source_name = reference['source_name']
1262
+ value = reference['external_id'].split('-')[1] if source_name == 'capec' else reference['url']
1263
+ attribute = deepcopy(stix2misp_mapping.attack_pattern_references_mapping[source_name]) if source_name in stix2misp_mapping.attack_pattern_references_mapping else stix2misp_mapping.references_attribute_mapping
1264
+ attribute['value'] = value
1265
+ misp_object.add_attribute(**attribute)
1266
+ self.fill_misp_object(misp_object, attack_pattern, 'attack_pattern_mapping')
1267
+ self.misp_event.add_object(**misp_object)
1268
+
1269
+ def parse_course_of_action(self, course_of_action):
1270
+ galaxy_names = self._check_existing_galaxy_name(course_of_action.name)
1271
+ if galaxy_names is not None:
1272
+ self.galaxy[course_of_action['id'].split('--')[1]] = {'tag_names': galaxy_names, 'used': False}
1273
+ else:
1274
+ misp_object = self.create_misp_object(course_of_action)
1275
+ self.fill_misp_object(misp_object, course_of_action, 'course_of_action_mapping')
1276
+ self.misp_event.add_object(**misp_object)
1277
+
1278
+ def parse_usual_indicator(self, indicator, separator):
1279
+ pattern = tuple(part.strip() for part in self._handle_pattern(indicator.pattern).split(separator))
1280
+ types = self._parse_pattern_types(pattern)
1281
+ try:
1282
+ getattr(self, stix2misp_mapping.pattern_mapping[types])(indicator, separator)
1283
+ except KeyError:
1284
+ print(f'Type(s) not supported at the moment: {types}\n', file=sys.stderr)
1285
+ self.add_stix2_pattern_object(indicator)
1286
+
1287
+ def parse_vulnerability(self, vulnerability):
1288
+ galaxy_names = self._check_existing_galaxy_name(vulnerability.name)
1289
+ if galaxy_names is not None:
1290
+ self.galaxy[vulnerability['id'].split('--')[1]] = {'tag_names': galaxy_names, 'used': False}
1291
+ else:
1292
+ attributes = self._get_attributes_from_observable(vulnerability, 'vulnerability_mapping')
1293
+ if hasattr(vulnerability, 'external_references'):
1294
+ for reference in vulnerability.external_references:
1295
+ if reference['source_name'] == 'url':
1296
+ attribute = deepcopy(stix2misp_mapping.references_attribute_mapping)
1297
+ attribute['value'] = reference['url']
1298
+ attributes.append(attribute)
1299
+ if len(attributes) == 1 and attributes[0]['object_relation'] == 'id':
1300
+ attributes[0]['type'] = 'vulnerability'
1301
+ self.handle_import_case(vulnerability, attributes, 'vulnerability')
1302
+
1303
+ ################################################################################
1304
+ ## OBSERVABLE PARSING FUNCTIONS ##
1305
+ ################################################################################
1306
+
1307
+ @staticmethod
1308
+ def _fetch_reference_type(references, object_type):
1309
+ for key, reference in references.items():
1310
+ if isinstance(reference, getattr(stix2.v20.observables, object_type)):
1311
+ return key
1312
+ return None
1313
+
1314
+ @staticmethod
1315
+ def _fetch_user_account_type_observable(observable_objects):
1316
+ for observable_object in observable_objects.values():
1317
+ if hasattr(observable_object, 'extensions') or any(key not in ('user_id', 'credential', 'type') for key in observable_object):
1318
+ return 'user-account', 'user_account_mapping'
1319
+ return 'credential', 'credential_mapping'
1320
+
1321
+ @staticmethod
1322
+ def _get_attributes_from_observable(stix_object, mapping):
1323
+ attributes = []
1324
+ for key, value in stix_object.items():
1325
+ if key in getattr(stix2misp_mapping, mapping) and value:
1326
+ attribute = deepcopy(getattr(stix2misp_mapping, mapping)[key])
1327
+ attribute.update({'value': value, 'to_ids': False})
1328
+ attributes.append(attribute)
1329
+ return attributes
1330
+
1331
+ def get_network_traffic_attributes(self, network_traffic, references):
1332
+ attributes = self._get_attributes_from_observable(network_traffic, 'network_traffic_mapping')
1333
+ mapping = 'network_traffic_references_mapping'
1334
+ attributes.extend(self.parse_network_traffic_references(network_traffic, references, mapping))
1335
+ if references:
1336
+ for reference in references.values():
1337
+ attributes.append(self._parse_observable_reference(reference, mapping, 'dst'))
1338
+ return attributes
1339
+
1340
+ @staticmethod
1341
+ def _handle_attachment_type(stix_object, is_reference, filename):
1342
+ _has_md5 = hasattr(stix_object, 'hashes') and 'MD5' in stix_object.hashes
1343
+ if is_reference and _has_md5:
1344
+ return 'malware-sample', f'{filename}|{stix_object.hashes["MD5"]}'
1345
+ return 'attachment', filename
1346
+
1347
+ def handle_pe_observable(self, attributes, extension, observable):
1348
+ pe_uuid = self.parse_pe(extension)
1349
+ file = self.create_misp_object(observable, 'file')
1350
+ file.add_reference(pe_uuid, 'includes')
1351
+ for attribute in attributes:
1352
+ file.add_attribute(**attribute)
1353
+ self.misp_event.add_object(file)
1354
+
1355
+ @staticmethod
1356
+ def _is_reference(network_traffic, reference):
1357
+ for feature in ('src', 'dst'):
1358
+ for reference_type in (f'{feature}_{ref}' for ref in ('ref', 'refs')):
1359
+ if reference in network_traffic.get(reference_type, []):
1360
+ return True
1361
+ return False
1362
+
1363
+ @staticmethod
1364
+ def _network_traffic_has_extension(network_traffic):
1365
+ if not hasattr(network_traffic, 'extensions'):
1366
+ return None
1367
+ if 'socket-ext' in network_traffic.extensions:
1368
+ return 'parse_socket_extension_observable'
1369
+ return None
1370
+
1371
+ def parse_asn_observable(self, observable):
1372
+ autonomous_system, references = self.filter_main_object(observable.objects, 'AutonomousSystem')
1373
+ mapping = 'asn_mapping'
1374
+ attributes = self._get_attributes_from_observable(autonomous_system, mapping)
1375
+ if references:
1376
+ for reference in references.values():
1377
+ attributes.append(self._parse_observable_reference(reference, mapping))
1378
+ self.handle_import_case(observable, attributes, 'asn')
1379
+
1380
+ def parse_domain_ip_observable(self, observable):
1381
+ domain, references = self.filter_main_object(observable.objects, 'DomainName')
1382
+ mapping = 'domain_ip_mapping'
1383
+ attributes = [self._parse_observable_reference(domain, mapping)]
1384
+ if references:
1385
+ for reference in references.values():
1386
+ attributes.append(self._parse_observable_reference(reference, mapping))
1387
+ self.handle_import_case(observable, attributes, 'domain-ip')
1388
+
1389
+ def parse_domain_ip_network_traffic_observable(self, observable):
1390
+ network_traffic, references = self.filter_main_object(observable.objects, 'NetworkTraffic')
1391
+ extension = self._network_traffic_has_extension(network_traffic)
1392
+ if extension:
1393
+ attributes, object_name = getattr(self, extension)(network_traffic, references)
1394
+ return self.handle_import_case(observable, attributes, object_name)
1395
+ if self._required_protocols(network_traffic.protocols):
1396
+ attributes = self.parse_network_connection_object(network_traffic, references)
1397
+ return self.handle_import_case(observable, attributes, 'network-connection')
1398
+ attributes, object_name = self.parse_network_traffic_objects(network_traffic, references)
1399
+ self.handle_import_case(observable, attributes, object_name)
1400
+
1401
+ def parse_domain_network_traffic_observable(self, observable):
1402
+ network_traffic, references = self.filter_main_object(observable.objects, 'NetworkTraffic')
1403
+ extension = self._network_traffic_has_extension(network_traffic)
1404
+ if extension:
1405
+ attributes, object_name = getattr(self, extension)(network_traffic, references)
1406
+ return self.handle_import_case(observable, attributes, object_name)
1407
+ attributes = self.parse_network_connection_object(network_traffic, references)
1408
+ self.handle_import_case(observable, attributes, 'network-connection')
1409
+
1410
+ def parse_email_address_observable(self, observable):
1411
+ self.add_attributes_from_observable(observable, 'email-src', 'value')
1412
+
1413
+ def parse_email_observable(self, observable):
1414
+ email_message, references = self.filter_main_object(observable.objects, 'EmailMessage')
1415
+ attributes = self._get_attributes_from_observable(email_message, 'email_mapping')
1416
+ if hasattr(email_message, 'additional_header_fields'):
1417
+ attributes.extend(self._get_attributes_from_observable(email_message.additional_header_fields, 'email_mapping'))
1418
+ attributes.extend(self._parse_email_references(email_message, references))
1419
+ if hasattr(email_message, 'body_multipart') and email_message.body_multipart:
1420
+ attributes.extend(self._parse_email_body(email_message.body_multipart, references))
1421
+ if references:
1422
+ print(f'Unable to parse the following observable objects: {references}', file=sys.stderr)
1423
+ self.handle_import_case(observable, attributes, 'email')
1424
+
1425
+ def parse_file_observable(self, observable):
1426
+ file_object, references = self.filter_main_object(observable.objects, 'File')
1427
+ attributes = self._get_attributes_from_observable(file_object, 'file_mapping')
1428
+ if 'hashes' in file_object:
1429
+ attributes.extend(self._get_attributes_from_observable(file_object.hashes, 'file_mapping'))
1430
+ if references:
1431
+ filename = file_object.name if hasattr(file_object, 'name') else 'unknown_filename'
1432
+ for key, reference in references.items():
1433
+ if isinstance(reference, stix2.v20.observables.Artifact):
1434
+ _is_content_ref = 'content_ref' in file_object and file_object.content_ref == key
1435
+ attribute_type, value = self._handle_attachment_type(reference, _is_content_ref, filename)
1436
+ attribute = {
1437
+ 'type': attribute_type,
1438
+ 'object_relation': attribute_type,
1439
+ 'value': value,
1440
+ 'to_ids': False
1441
+ }
1442
+ if hasattr(reference, 'payload_bin'):
1443
+ attribute['data'] = reference.payload_bin
1444
+ attributes.append(attribute)
1445
+ elif isinstance(reference, stix2.v20.observables.Directory):
1446
+ attribute = {
1447
+ 'type': 'text',
1448
+ 'object_relation': 'path',
1449
+ 'value': reference.path,
1450
+ 'to_ids': False
1451
+ }
1452
+ attributes.append(attribute)
1453
+ if hasattr(file_object, 'extensions'):
1454
+ # Support of more extension types probably in the future
1455
+ if 'windows-pebinary-ext' in file_object.extensions:
1456
+ # Here we do not go to the standard route of "handle_import_case"
1457
+ # because we want to make sure a file object is created
1458
+ return self.handle_pe_observable(attributes, file_object.extensions['windows-pebinary-ext'], observable)
1459
+ extension_types = (extension_type for extension_type in file_object.extensions.keys())
1460
+ print(f'File extension type(s) not supported at the moment: {", ".join(extension_types)}', file=sys.stderr)
1461
+ self.handle_import_case(observable, attributes, 'file', _force_object=('file-encoding', 'path'))
1462
+
1463
+ def parse_ip_address_observable(self, observable):
1464
+ attributes = []
1465
+ for observable_object in observable.objects.values():
1466
+ attribute = {
1467
+ 'value': observable_object.value,
1468
+ 'to_ids': False
1469
+ }
1470
+ attribute.update(stix2misp_mapping.ip_attribute_mapping)
1471
+ attributes.append(attribute)
1472
+ self.handle_import_case(observable, attributes, 'ip-port')
1473
+
1474
+ def parse_ip_network_traffic_observable(self, observable):
1475
+ network_traffic, references = self.filter_main_object(observable.objects, 'NetworkTraffic')
1476
+ extension = self._network_traffic_has_extension(network_traffic)
1477
+ if extension:
1478
+ attributes, object_name = getattr(self, extension)(network_traffic, references)
1479
+ return self.handle_import_case(observable, attributes, object_name)
1480
+ attributes = self.parse_ip_port_object(network_traffic, references)
1481
+ self.handle_import_case(observable, attributes, 'ip-port')
1482
+
1483
+ def parse_ip_port_object(self, network_traffic, references):
1484
+ attributes = self._get_attributes_from_observable(network_traffic, 'network_traffic_mapping')
1485
+ attributes.extend(self.parse_network_traffic_references(network_traffic, references, 'ip_port_references_mapping'))
1486
+ if references:
1487
+ for reference in references.values():
1488
+ attributes.append(self._parse_observable_reference(reference, 'domain_ip_mapping'))
1489
+ return attributes
1490
+
1491
+ def parse_mac_address_observable(self, observable):
1492
+ self.add_attributes_from_observable(observable, 'mac-address', 'value')
1493
+
1494
+ def parse_network_connection_object(self, network_traffic, references):
1495
+ attributes = self.get_network_traffic_attributes(network_traffic, references)
1496
+ attributes.extend(self.parse_protocols(network_traffic.protocols, 'observable object'))
1497
+ return attributes
1498
+
1499
+ def parse_network_traffic_objects(self, network_traffic, references):
1500
+ _has_domain = self._fetch_reference_type(references.values(), 'DomainName')
1501
+ if _has_domain and self._is_reference(network_traffic, _has_domain):
1502
+ return self.parse_network_connection_object(network_traffic, references), 'network-connection'
1503
+ return self.parse_ip_port_object(network_traffic, references), 'ip-port'
1504
+
1505
+ def parse_network_traffic_references(self, network_traffic, references, mapping):
1506
+ attributes = []
1507
+ for feature in ('src', 'dst'):
1508
+ ref = f'{feature}_ref'
1509
+ if hasattr(network_traffic, ref):
1510
+ reference = getattr(network_traffic, ref)
1511
+ attributes.append(self._parse_observable_reference(references.pop(reference), mapping, feature))
1512
+ if hasattr(network_traffic, f'{ref}s'):
1513
+ for reference in getattr(network_traffic, f'{ref}s'):
1514
+ attributes.append(self._parse_observable_reference(references.pop(reference), mapping, feature))
1515
+ return attributes
1516
+
1517
+ def parse_mutex_observable(self, observable):
1518
+ self.add_attributes_from_observable(observable, 'mutex', 'name')
1519
+
1520
+ def parse_process_observable(self, observable):
1521
+ process, references = self.filter_main_object(observable.objects, 'Process', test_function='_process_test_filter')
1522
+ attributes = self._get_attributes_from_observable(process, 'process_mapping')
1523
+ if hasattr(process, 'parent_ref'):
1524
+ attributes.extend(self._get_attributes_from_observable(references.pop(process.parent_ref), 'parent_process_reference_mapping'))
1525
+ if hasattr(process, 'child_refs'):
1526
+ for reference in process.child_refs:
1527
+ attributes.extend(self._get_attributes_from_observable(references.pop(reference), 'child_process_reference_mapping'))
1528
+ if hasattr(process, 'binary_ref'):
1529
+ reference = references.pop(process.binary_ref)
1530
+ attribute = {
1531
+ 'value': reference.name,
1532
+ 'to_ids': False
1533
+ }
1534
+ attribute.update(stix2misp_mapping.process_image_mapping)
1535
+ attributes.append(attribute)
1536
+ if references:
1537
+ print(f'Unable to parse the following observable objects: {references}', file=sys.stderr)
1538
+ self.handle_import_case(observable, attributes, 'process', _force_object=True)
1539
+
1540
+ def parse_protocols(self, protocols, object_type):
1541
+ attributes = []
1542
+ protocols = (protocol.upper() for protocol in protocols)
1543
+ for protocol in protocols:
1544
+ try:
1545
+ attributes.append(self._parse_network_traffic_protocol(protocol))
1546
+ except KeyError:
1547
+ print(f'Unknown protocol in network-traffic {object_type}: {protocol}', file=sys.stderr)
1548
+ return attributes
1549
+
1550
+ def parse_regkey_observable(self, observable):
1551
+ attributes = []
1552
+ for observable_object in observable.objects.values():
1553
+ attributes.extend(self._get_attributes_from_observable(observable_object, 'regkey_mapping'))
1554
+ if 'values' in observable_object:
1555
+ for registry_value in observable_object['values']:
1556
+ attributes.extend(self._get_attributes_from_observable(registry_value, 'regkey_mapping'))
1557
+ self.handle_import_case(observable, attributes, 'registry-key')
1558
+
1559
+ def parse_socket_extension_observable(self, network_traffic, references):
1560
+ attributes = self.get_network_traffic_attributes(network_traffic, references)
1561
+ for key, value in network_traffic.extensions['socket-ext'].items():
1562
+ if key not in stix2misp_mapping.network_socket_extension_mapping:
1563
+ print(f'Unknown socket extension field in observable object: {key}', file=sys.stderr)
1564
+ continue
1565
+ if key.startswith('is_') and not value:
1566
+ continue
1567
+ attribute = {
1568
+ 'value': key.split('_')[1] if key.startswith('is_') else value,
1569
+ 'to_ids': False
1570
+ }
1571
+ attribute.update(stix2misp_mapping.network_socket_extension_mapping[key])
1572
+ attributes.append(attribute)
1573
+ return attributes, 'network-socket'
1574
+
1575
+ def parse_url_observable(self, observable):
1576
+ network_traffic, references = self.filter_main_object(observable.objects, 'NetworkTraffic')
1577
+ attributes = self._get_attributes_from_observable(network_traffic, 'network_traffic_mapping') if network_traffic else []
1578
+ if references:
1579
+ for reference in references.values():
1580
+ attributes.append(self._parse_observable_reference(reference, 'url_mapping'))
1581
+ self.handle_import_case(observable, attributes, 'url')
1582
+
1583
+ def parse_user_account_extension(self, extension):
1584
+ attributes = self._parse_user_account_groups(extension['groups']) if 'groups' in extension else []
1585
+ attributes.extend(self._get_attributes_from_observable(extension, 'user_account_mapping'))
1586
+ return attributes
1587
+
1588
+ def parse_user_account_observable(self, observable):
1589
+ attributes = []
1590
+ object_name, mapping = self._fetch_user_account_type_observable(observable.objects)
1591
+ for observable_object in observable.objects.values():
1592
+ attributes.extend(self._get_attributes_from_observable(observable_object, mapping))
1593
+ if hasattr(observable_object, 'extensions') and observable_object.extensions.get('unix-account-ext'):
1594
+ attributes.extend(self.parse_user_account_extension(observable_object.extensions['unix-account-ext']))
1595
+ self.handle_import_case(observable, attributes, object_name)
1596
+
1597
+ def parse_x509_observable(self, observable):
1598
+ attributes = []
1599
+ for observable_object in observable.objects.values():
1600
+ attributes.extend(self._get_attributes_from_observable(observable_object, 'x509_mapping'))
1601
+ if hasattr(observable_object, 'hashes'):
1602
+ attributes.extend(self._get_attributes_from_observable(observable_object.hashes, 'x509_mapping'))
1603
+ self.handle_import_case(observable, attributes, 'x509')
1604
+
1605
+ ################################################################################
1606
+ ## PATTERN PARSING FUNCTIONS. ##
1607
+ ################################################################################
1608
+
1609
+ @staticmethod
1610
+ def _fetch_user_account_type_pattern(pattern):
1611
+ for stix_object in pattern:
1612
+ if 'extensions' in stix_object or all(key not in stix_object for key in ('user_id', 'credential', 'type')):
1613
+ return 'user-account', 'user_account_mapping'
1614
+ return 'credential', 'credential_mapping'
1615
+
1616
+ def get_attachment(self, attachment, filename):
1617
+ attribute = {
1618
+ 'type': 'attachment',
1619
+ 'object_relation': 'attachment',
1620
+ 'value': attachment.pop(filename)
1621
+ }
1622
+ data_feature = self._choose_with_priority(attachment, 'file:content_ref.payload_bin', 'artifact:payload_bin')
1623
+ attribute['data'] = attachment.pop(data_feature)
1624
+ return attribute
1625
+
1626
+ def get_attributes_from_pattern(self, pattern, mapping, separator):
1627
+ attributes = []
1628
+ for pattern_part in pattern.strip('[]').split(separator):
1629
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1630
+ try:
1631
+ attribute = deepcopy(getattr(stix2misp_mapping, mapping)[pattern_type])
1632
+ except KeyError:
1633
+ print(f'Pattern type not supported at the moment: {pattern_type}', file=sys.stderr)
1634
+ continue
1635
+ attribute['value'] = pattern_value
1636
+ attributes.append(attribute)
1637
+ return attributes
1638
+
1639
+ def get_malware_sample(self, attachment, filename):
1640
+ md5_feature = self._choose_with_priority(attachment, "file:content_ref.hashes.'MD5'", "file:hashes.'MD5'")
1641
+ attribute = {
1642
+ 'type': 'malware-sample',
1643
+ 'object_relation': 'malware-sample',
1644
+ 'value': f'{attachment.pop(filename)}|{attachment.pop(md5_feature)}'
1645
+ }
1646
+ data_feature = self._choose_with_priority(attachment, 'file:content_ref.payload_bin', 'artifact:payload_bin')
1647
+ attribute['data'] = attachment.pop(data_feature)
1648
+ return attribute
1649
+
1650
+ def _handle_file_attachments(self, attachment):
1651
+ attributes = []
1652
+ if any('content_ref' in feature for feature in attachment.keys()):
1653
+ attribute_type = 'attachment'
1654
+ value = attachment['file:name'] if 'file:name' in attachment else 'unknown_filename'
1655
+ if "file:content_ref.hashes.'MD5'" in attachment:
1656
+ attribute_type = 'malware-sample'
1657
+ md5 = attachment.pop("file:content_ref.hashes.'MD5'")
1658
+ value = f'{value}|{md5}'
1659
+ data = self._choose_with_priority(attachment, 'file:content_ref.payload_bin', 'artifact:payload_bin')
1660
+ attribute = {
1661
+ 'type': attribute_type,
1662
+ 'object_relation': attribute_type,
1663
+ 'value': value,
1664
+ 'data': attachment.pop(data)
1665
+ }
1666
+ attributes.append(attribute)
1667
+ if 'artifact:payload_bin' in attachment:
1668
+ attribute = {
1669
+ 'type': 'attachment',
1670
+ 'object_relation': 'attachment',
1671
+ 'value': attachment['file:name'],
1672
+ 'data': attachment.pop('artifact:payload_bin')
1673
+ }
1674
+ attributes.append(attribute)
1675
+ return attributes
1676
+
1677
+ def parse_as_pattern(self, indicator, separator):
1678
+ attributes = self.get_attributes_from_pattern(indicator.pattern, 'asn_mapping', separator)
1679
+ self.handle_import_case(indicator, attributes, 'asn')
1680
+
1681
+ def parse_domain_ip_port_pattern(self, indicator, separator):
1682
+ attributes = []
1683
+ references = defaultdict(dict)
1684
+ for pattern_part in self._handle_pattern(indicator.pattern).split(separator):
1685
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1686
+ if pattern_type not in stix2misp_mapping.domain_ip_mapping:
1687
+ if any(pattern_type.startswith(f'network-traffic:{feature}_ref') for feature in ('src', 'dst')):
1688
+ feature_type, ref = pattern_type.split(':')[1].split('_')
1689
+ ref, feature = ref.split('.')
1690
+ ref = f"{feature_type}_{'0' if ref == 'ref' else ref.strip('ref[]')}"
1691
+ references[ref].update(self._parse_network_connection_reference(feature_type, feature, pattern_value))
1692
+ else:
1693
+ print(f'Pattern type not currently mapped: {pattern_type}', file=sys.stderr)
1694
+ continue
1695
+ attribute = deepcopy(stix2misp_mapping.domain_ip_mapping[pattern_type])
1696
+ attribute['value'] = pattern_value
1697
+ attributes.append(attribute)
1698
+ if references:
1699
+ attributes.extend(references.values())
1700
+ object_name = 'ip-port' if 'network-traffic' in indicator.pattern else 'domain-ip'
1701
+ self.handle_import_case(indicator, attributes, object_name)
1702
+
1703
+ def parse_email_address_pattern(self, indicator, separator):
1704
+ self.add_attributes_from_indicator(indicator, 'email-src', separator)
1705
+
1706
+ def parse_email_message_pattern(self, indicator, separator):
1707
+ attributes = []
1708
+ attachments = defaultdict(dict)
1709
+ for pattern_part in self._handle_pattern(indicator.pattern).split(separator):
1710
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1711
+ if pattern_type not in stix2misp_mapping.email_mapping:
1712
+ if pattern_type.startswith('email-message:body_multipart'):
1713
+ features = pattern_type.split('.')
1714
+ if len(features) == 3 and features[1] == 'body_raw_ref':
1715
+ index = features[0].split('[')[1].strip(']') if '[' in features[0] else '0'
1716
+ key = 'data' if features[2] == 'payload_bin' else 'value'
1717
+ attachments[index][key] = pattern_value
1718
+ continue
1719
+ print(f'Pattern type not supported at the moment: {pattern_type}', file=sys.stderr)
1720
+ continue
1721
+ attribute = deepcopy(stix2misp_mapping.email_mapping[pattern_type])
1722
+ attribute['value'] = pattern_value
1723
+ attributes.append(attribute)
1724
+ if attachments:
1725
+ for attachment in attachments.values():
1726
+ attribute = {
1727
+ 'type': 'attachment',
1728
+ 'object_relation': 'screenshot'
1729
+ } if 'data' in attachment else {
1730
+ 'type': 'email-attachment',
1731
+ 'object_relation': 'attachment'
1732
+ }
1733
+ attribute.update(attachment)
1734
+ attributes.append(attribute)
1735
+ self.handle_import_case(indicator, attributes, 'email')
1736
+
1737
+ def parse_file_pattern(self, indicator, separator):
1738
+ attributes = []
1739
+ attachment = {}
1740
+ extensions = defaultdict(lambda: defaultdict(dict))
1741
+ for pattern_part in self._handle_pattern(indicator.pattern).split(separator):
1742
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1743
+ if pattern_type in stix2misp_mapping.attachment_types:
1744
+ attachment[pattern_type] = pattern_value.strip("'")
1745
+ continue
1746
+ if pattern_type not in stix2misp_mapping.file_mapping:
1747
+ if 'extensions' in pattern_type:
1748
+ features = pattern_type.split('.')[1:]
1749
+ extension_type = features.pop(0).strip("'")
1750
+ if 'section' in features[0] and features[0] != 'number_of_sections':
1751
+ index = features[0].split('[')[1].strip(']') if '[' in features[0] else '0'
1752
+ extensions[extension_type][f'section_{index}'][features[-1].strip("'")] = pattern_value
1753
+ else:
1754
+ extensions[extension_type]['.'.join(features)] = pattern_value
1755
+ continue
1756
+ attribute = deepcopy(stix2misp_mapping.file_mapping[pattern_type])
1757
+ attribute['value'] = pattern_value
1758
+ attributes.append(attribute)
1759
+ if any(key.endswith('payload_bin') for key in attachment.keys()):
1760
+ attributes.extend(self._handle_file_attachments(attachment))
1761
+ if attachment:
1762
+ for pattern_type, value in attachment.items():
1763
+ if pattern_type in stix2misp_mapping.file_mapping:
1764
+ attribute = deepcopy(stix2misp_mapping.file_mapping[pattern_type])
1765
+ attribute['value'] = value
1766
+ attributes.append(attribute)
1767
+ if extensions:
1768
+ file_object = self.create_misp_object(indicator, 'file')
1769
+ self.parse_file_extension(file_object, attributes, extensions)
1770
+ else:
1771
+ self.handle_import_case(indicator, attributes, 'file', _force_object=('file-encoding', 'path'))
1772
+
1773
+ def parse_file_extension(self, file_object, attributes, extensions):
1774
+ for attribute in attributes:
1775
+ file_object.add_attribute(**attribute)
1776
+ if 'windows-pebinary-ext' in extensions:
1777
+ pe_extension = extensions['windows-pebinary-ext']
1778
+ pe_object = MISPObject('pe', misp_objects_path_custom=_misp_objects_path)
1779
+ sections = self._get_sections(pe_extension)
1780
+ self.fill_misp_object_from_dict(pe_object, pe_extension, 'pe_mapping')
1781
+ if sections:
1782
+ for section in sections:
1783
+ section_object = MISPObject('pe-section')
1784
+ self.fill_misp_object_from_dict(section_object, section, 'pe_section_mapping')
1785
+ self.misp_event.add_object(section_object)
1786
+ pe_object.add_reference(section_object.uuid, 'includes')
1787
+ self.misp_event.add_object(pe_object)
1788
+ file_object.add_reference(pe_object.uuid, 'includes')
1789
+ self.misp_event.add_object(file_object)
1790
+
1791
+ def parse_ip_address_pattern(self, indicator, separator):
1792
+ self.add_attributes_from_indicator(indicator, 'ip-dst', separator)
1793
+
1794
+ def parse_mac_address_pattern(self, indicator, separator):
1795
+ self.add_attributes_from_indicator(indicator, 'mac-address', separator)
1796
+
1797
+ def parse_mutex_pattern(self, indicator, separator):
1798
+ self.add_attributes_from_indicator(indicator, 'mutex', separator)
1799
+
1800
+ def parse_network_connection_pattern(self, indicator, attributes, references):
1801
+ attributes.extend(self._parse_network_pattern_references(references, 'network_traffic_references_mapping'))
1802
+ self.handle_import_case(indicator, attributes, 'network-connection')
1803
+
1804
+ @staticmethod
1805
+ def _parse_network_pattern_references(references, mapping):
1806
+ attributes = []
1807
+ for feature, reference in references.items():
1808
+ feature = feature.split('_')[0]
1809
+ attribute = {key: value.format(feature) for key, value in getattr(stix2misp_mapping, mapping)[reference['type']].items()}
1810
+ attribute['value'] = reference['value']
1811
+ attributes.append(attribute)
1812
+ return attributes
1813
+
1814
+ def parse_network_socket_pattern(self, indicator, attributes, references, extension):
1815
+ attributes.extend(self._parse_network_pattern_references(references, 'network_traffic_references_mapping'))
1816
+ for key, value in extension.items():
1817
+ if key not in stix2misp_mapping.network_socket_extension_mapping:
1818
+ print(f'Unknown socket extension field in pattern: {key}', file=sys.stderr)
1819
+ continue
1820
+ if key.startswith('is_') and not json.loads(value.lower()):
1821
+ continue
1822
+ attribute = deepcopy(stix2misp_mapping.network_socket_extension_mapping[key])
1823
+ attribute['value'] = key.split('_')[1] if key.startswith('is_') else value
1824
+ attributes.append(attribute)
1825
+ self.handle_import_case(indicator, attributes, 'network-socket')
1826
+
1827
+ def parse_network_traffic_pattern(self, indicator, separator):
1828
+ attributes = []
1829
+ protocols = []
1830
+ references = defaultdict(dict)
1831
+ extensions = defaultdict(dict)
1832
+ for pattern_part in self._handle_pattern(indicator.pattern).split(separator):
1833
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1834
+ if pattern_type in stix2misp_mapping.network_traffic_mapping:
1835
+ attribute = deepcopy(stix2misp_mapping.network_traffic_mapping[pattern_type])
1836
+ attribute['value'] = pattern_value.strip("'")
1837
+ attributes.append(attribute)
1838
+ continue
1839
+ if pattern_type.startswith('network-traffic:protocols['):
1840
+ protocols.append(pattern_value)
1841
+ elif any(pattern_type.startswith(f'network-traffic:{feature}_ref') for feature in ('src', 'dst')):
1842
+ feature_type, ref = pattern_type.split(':')[1].split('_')
1843
+ ref, feature = ref.split('.')
1844
+ ref = f"{feature_type}_{'0' if ref == 'ref' else ref.strip('ref[]')}"
1845
+ references[ref].update({feature: pattern_value})
1846
+ elif pattern_type.startswith('network-traffic:extensions.'):
1847
+ _, extension_type, feature = pattern_type.split('.')
1848
+ extensions[extension_type.strip("'")][feature] = pattern_value
1849
+ else:
1850
+ print(f'Pattern type not supported at the moment: {pattern_type}', file=sys.stderr)
1851
+ if extensions:
1852
+ if 'socket-ext' in extensions:
1853
+ return self.parse_network_socket_pattern(indicator, attributes, references, extensions['socket-ext'])
1854
+ print(f'Unknown network extension(s) in pattern: {", ".join(extensions.keys())}', file=sys.stderr)
1855
+ if protocols and self._required_protocols(protocols):
1856
+ attributes.extend(self.parse_protocols(protocols, 'pattern'))
1857
+ return self.parse_network_connection_pattern(indicator, attributes, references)
1858
+ attributes.extend(self._parse_network_pattern_references(references, 'ip_port_references_mapping'))
1859
+ self.handle_import_case(indicator, attributes, 'ip-port')
1860
+
1861
+ def parse_process_pattern(self, indicator, separator):
1862
+ attributes = []
1863
+ parent = {}
1864
+ child = defaultdict(set)
1865
+ for pattern_part in self._handle_pattern(indicator.pattern).split(separator):
1866
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1867
+ if 'parent_' in pattern_type:
1868
+ child[pattern_type.split('.')[-1]].add(pattern_value)
1869
+ elif 'child_' in pattern_type:
1870
+ parent[pattern_type.split('.')[-1]] = pattern_value
1871
+ else:
1872
+ try:
1873
+ attribute = deepcopy(stix2misp_mapping.process_mapping[pattern_type])
1874
+ except KeyError:
1875
+ print(f'Pattern type not supported at the moment: {pattern_type}', file=sys.stderr)
1876
+ continue
1877
+ attribute['value'] = pattern_value
1878
+ attributes.append(attribute)
1879
+ if parent:
1880
+ for key, value in parent.items():
1881
+ if key not in stix2misp_mapping.parent_process_reference_mapping:
1882
+ print(f'Parent process key from pattern not supported at the moment: {key}', file=sys.stderr)
1883
+ continue
1884
+ attribute = {'value': value}
1885
+ attribute.update(stix2misp_mapping.parent_process_reference_mapping[key])
1886
+ attributes.append(attribute)
1887
+ if child:
1888
+ for key, values in child.items():
1889
+ if key not in stix2misp_mapping.child_process_reference_mapping:
1890
+ print(f'Child process key from pattern not supported at the moment: {key}', file=sys.stderr)
1891
+ continue
1892
+ for value in values:
1893
+ attribute = {'value': value}
1894
+ attribute.update(stix2misp_mapping.child_process_reference_mapping[key])
1895
+ attributes.append(attribute)
1896
+ self.handle_import_case(indicator, attributes, 'process', _force_object=True)
1897
+
1898
+ def parse_regkey_pattern(self, indicator, separator):
1899
+ attributes = self.get_attributes_from_pattern(indicator.pattern, 'regkey_mapping', separator)
1900
+ self.handle_import_case(indicator, attributes, 'registry-key')
1901
+
1902
+ def parse_url_pattern(self, indicator, separator):
1903
+ attributes = self.get_attributes_from_pattern(indicator.pattern, 'url_mapping', separator)
1904
+ self.handle_import_case(indicator, attributes, 'url')
1905
+
1906
+ def parse_user_account_pattern(self, indicator, separator):
1907
+ attributes = []
1908
+ pattern = self._handle_pattern(indicator.pattern).split(separator)
1909
+ object_name, mapping = self._fetch_user_account_type_pattern(pattern)
1910
+ for pattern_part in pattern:
1911
+ pattern_type, pattern_value = self.get_type_and_value_from_pattern(pattern_part)
1912
+ pattern_type = pattern_type.split(':')[1]
1913
+ if pattern_type.startswith('extensions.'):
1914
+ pattern_type = pattern_type.split('.')[-1]
1915
+ if '[' in pattern_type:
1916
+ pattern_type = pattern_type.split('[')[0]
1917
+ if pattern_type in ('group', 'groups'):
1918
+ attributes.append({'type': 'text', 'object_relation': 'group', 'value': pattern_value})
1919
+ continue
1920
+ if pattern_type in getattr(stix2misp_mapping, mapping):
1921
+ attribute = deepcopy(getattr(stix2misp_mapping, mapping)[pattern_type])
1922
+ attribute['value'] = pattern_value
1923
+ attributes.append(attribute)
1924
+ self.handle_import_case(indicator, attributes, object_name)
1925
+
1926
+ def parse_x509_pattern(self, indicator, separator):
1927
+ attributes = self.get_attributes_from_pattern(indicator.pattern, 'x509_mapping', separator)
1928
+ self.handle_import_case(indicator, attributes, 'x509')
1929
+
1930
+ ################################################################################
1931
+ ## UTILITY FUNCTIONS. ##
1932
+ ################################################################################
1933
+
1934
+ def add_attributes_from_indicator(self, indicator, attribute_type, separator):
1935
+ patterns = self._handle_pattern(indicator.pattern).split(separator)
1936
+ if len(patterns) == 1:
1937
+ _, value = self.get_type_and_value_from_pattern(patterns[0])
1938
+ attribute = MISPAttribute()
1939
+ attribute.from_dict(**{
1940
+ 'uuid': indicator.id.split('--')[1],
1941
+ 'type': attribute_type,
1942
+ 'value': value,
1943
+ 'to_ids': True
1944
+ })
1945
+ attribute.update(self.parse_timeline(indicator))
1946
+ self.misp_event.add_attribute(**attribute)
1947
+ else:
1948
+ tmp_attribute = self.parse_timeline(indicator)
1949
+ for pattern in patterns:
1950
+ _, value = self.get_type_and_value_from_pattern(pattern)
1951
+ attribute = MISPAttribute()
1952
+ attribute.from_dict(**{
1953
+ 'type': attribute_type,
1954
+ 'value': value,
1955
+ 'to_ids': True
1956
+ })
1957
+ attribute.update(tmp_attribute)
1958
+ self.misp_event.add_attribute(**attribute)
1959
+
1960
+ def add_attributes_from_observable(self, observable, attribute_type, feature):
1961
+ if len(observable.objects) == 1:
1962
+ attribute = MISPAttribute()
1963
+ attribute.from_dict(**{
1964
+ 'uuid': observable.id.split('--')[1],
1965
+ 'type': attribute_type,
1966
+ 'value': getattr(observable.objects['0'], feature),
1967
+ 'to_ids': False
1968
+ })
1969
+ attribute.update(self.parse_timeline(observable))
1970
+ self.misp_event.add_attribute(**attribute)
1971
+ else:
1972
+ tmp_attribute = self.parse_timeline(observable)
1973
+ for observable_object in observable.objects.values():
1974
+ attribute = MISPAttribute()
1975
+ attribute.from_dict(**{
1976
+ 'type': attribute_type,
1977
+ 'value': getattr(observable_object, feature),
1978
+ 'to_ids': False
1979
+ })
1980
+ attribute.update(tmp_attribute)
1981
+ self.misp_event.add_attribute(**attribute)
1982
+
1983
+ def _check_existing_galaxy_name(self, galaxy_name):
1984
+ if galaxy_name in self._synonyms_to_tag_names:
1985
+ return self._synonyms_to_tag_names[galaxy_name]
1986
+ for name, tag_names in self._synonyms_to_tag_names.items():
1987
+ if galaxy_name in name:
1988
+ return tag_names
1989
+ return None
1990
+
1991
+ def create_misp_object(self, stix_object, name=None):
1992
+ misp_object = MISPObject(name if name is not None else stix_object.type,
1993
+ misp_objects_path_custom=_misp_objects_path)
1994
+ misp_object.uuid = stix_object.id.split('--')[1]
1995
+ if hasattr(stix_object, 'description') and stix_object.description:
1996
+ misp_object.comment = stix_object.description
1997
+ misp_object.update(self.parse_timeline(stix_object))
1998
+ return misp_object
1999
+
2000
+ @staticmethod
2001
+ def _get_sections(pe_extension):
2002
+ sections = [feature for feature in pe_extension.keys() if feature.startswith('section_')]
2003
+ return (pe_extension.pop(feature) for feature in sections)
2004
+
2005
+ @staticmethod
2006
+ def get_type_and_value_from_pattern(pattern):
2007
+ pattern = pattern.strip('[]')
2008
+ try:
2009
+ pattern_type, pattern_value = pattern.split(' = \'')
2010
+ except ValueError:
2011
+ pattern_type, pattern_value = pattern.split('=')
2012
+ return pattern_type.strip(), pattern_value.strip("'")
2013
+
2014
+ def handle_import_case(self, stix_object, attributes, name, _force_object=False):
2015
+ try:
2016
+ if len(attributes) > 1 or (_force_object and self._handle_object_forcing(_force_object, attributes[0])):
2017
+ misp_object = self.create_misp_object(stix_object, name)
2018
+ for attribute in attributes:
2019
+ misp_object.add_attribute(**attribute)
2020
+ self.misp_event.add_object(**misp_object)
2021
+ else:
2022
+ attribute = {field: attributes[0][field] for field in stix2misp_mapping.single_attribute_fields if attributes[0].get(field) is not None}
2023
+ attribute['uuid'] = stix_object.id.split('--')[1]
2024
+ attribute.update(self.parse_timeline(stix_object))
2025
+ if isinstance(stix_object, stix2.v20.Indicator):
2026
+ attribute['to_ids'] = True
2027
+ if hasattr(stix_object, 'object_marking_refs'):
2028
+ self.update_marking_refs(attribute['uuid'], stix_object.object_marking_refs)
2029
+ self.misp_event.add_attribute(**attribute)
2030
+ except IndexError:
2031
+ object_type = 'indicator' if isinstance(stix_object, stix2.Indicator) else 'observable objects'
2032
+ print(f'No attribute or object could be imported from the following {object_type}: {stix_object}', file=sys.stderr)
2033
+
2034
+ @staticmethod
2035
+ def _handle_object_forcing(_force_object, attribute):
2036
+ if isinstance(_force_object, (list, tuple)):
2037
+ return attribute['object_relation'] in _force_object
2038
+ return _force_object
2039
+
2040
+ @staticmethod
2041
+ def _handle_pattern(pattern):
2042
+ return pattern.strip().strip('[]')
2043
+
2044
+ @staticmethod
2045
+ def _parse_observable_types(observable_objects):
2046
+ types = {observable_object._type for observable_object in observable_objects.values()}
2047
+ return tuple(sorted(types))
2048
+
2049
+ @staticmethod
2050
+ def _parse_pattern_types(pattern):
2051
+ types = {part.split('=')[0].split(':')[0].strip('[') for part in pattern}
2052
+ return tuple(sorted(types))
2053
+
2054
+ @staticmethod
2055
+ def _required_protocols(protocols):
2056
+ protocols = tuple(protocol.upper() for protocol in protocols)
2057
+ if any(protocol not in ('TCP', 'IP') for protocol in protocols):
2058
+ return True
2059
+ return False
2060
+
2061
+
2062
+ def from_misp(stix_objects):
2063
+ for stix_object in stix_objects:
2064
+ if stix_object['type'] == "report" and 'misp:tool="misp2stix2"' in stix_object.get('labels', []):
2065
+ return True
2066
+ return False
2067
+
2068
+
2069
+ def main(args):
2070
+ filename = args[1] if args[1][0] == '/' else Path(os.path.dirname(args[0]), args[1])
2071
+ with open(filename, 'rt', encoding='utf-8') as f:
2072
+ event = stix2.parse(f.read(), allow_custom=True, interoperability=True)
2073
+ stix_parser = StixFromMISPParser() if from_misp(event.objects) else ExternalStixParser()
2074
+ stix_parser.handler(event, filename, args[2:])
2075
+ stix_parser.save_file()
2076
+ print(1)
2077
+
2078
+
2079
+ if __name__ == '__main__':
2080
+ main(sys.argv)