misp-modules 2.4.196__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. misp_modules/__init__.py +379 -0
  2. misp_modules/helpers/__init__.py +1 -0
  3. misp_modules/helpers/cache.py +84 -0
  4. misp_modules/lib/__init__.py +4 -0
  5. misp_modules/lib/_vmray/__init__.py +0 -0
  6. misp_modules/lib/_vmray/parser.py +1417 -0
  7. misp_modules/lib/_vmray/rest_api.py +148 -0
  8. misp_modules/lib/cof2misp/LICENSE-2.0.txt +202 -0
  9. misp_modules/lib/cof2misp/__init__.py +0 -0
  10. misp_modules/lib/cof2misp/cof.py +165 -0
  11. misp_modules/lib/joe_mapping.py +114 -0
  12. misp_modules/lib/joe_parser.py +573 -0
  13. misp_modules/lib/lastline_api.py +841 -0
  14. misp_modules/lib/qintel_helper.py +263 -0
  15. misp_modules/lib/stix2misp.py +2080 -0
  16. misp_modules/lib/stix2misp_mapping.py +460 -0
  17. misp_modules/lib/synonymsToTagNames.json +1 -0
  18. misp_modules/lib/vt_graph_parser/__init__.py +8 -0
  19. misp_modules/lib/vt_graph_parser/errors.py +20 -0
  20. misp_modules/lib/vt_graph_parser/helpers/__init__.py +7 -0
  21. misp_modules/lib/vt_graph_parser/helpers/parsers.py +88 -0
  22. misp_modules/lib/vt_graph_parser/helpers/rules.py +304 -0
  23. misp_modules/lib/vt_graph_parser/helpers/wrappers.py +58 -0
  24. misp_modules/lib/vt_graph_parser/importers/__init__.py +7 -0
  25. misp_modules/lib/vt_graph_parser/importers/base.py +98 -0
  26. misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +73 -0
  27. misp_modules/modules/__init__.py +4 -0
  28. misp_modules/modules/action_mod/__init__.py +1 -0
  29. misp_modules/modules/action_mod/_utils/__init__.py +1 -0
  30. misp_modules/modules/action_mod/_utils/utils.py +70 -0
  31. misp_modules/modules/action_mod/mattermost.py +113 -0
  32. misp_modules/modules/action_mod/slack.py +96 -0
  33. misp_modules/modules/action_mod/testaction.py +68 -0
  34. misp_modules/modules/expansion/__init__.py +36 -0
  35. misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT +27 -0
  36. misp_modules/modules/expansion/_dnsdb_query/LICENSE +202 -0
  37. misp_modules/modules/expansion/_dnsdb_query/README.md +162 -0
  38. misp_modules/modules/expansion/_dnsdb_query/__init__.py +0 -0
  39. misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +327 -0
  40. misp_modules/modules/expansion/_ransomcoindb/__init__.py +0 -0
  41. misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py +96 -0
  42. misp_modules/modules/expansion/_vulnerability_parser/__init__.py +0 -0
  43. misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py +118 -0
  44. misp_modules/modules/expansion/abuseipdb.py +146 -0
  45. misp_modules/modules/expansion/apiosintds.py +365 -0
  46. misp_modules/modules/expansion/apivoid.py +133 -0
  47. misp_modules/modules/expansion/assemblyline_query.py +179 -0
  48. misp_modules/modules/expansion/assemblyline_submit.py +100 -0
  49. misp_modules/modules/expansion/backscatter_io.py +84 -0
  50. misp_modules/modules/expansion/btc_scam_check.py +54 -0
  51. misp_modules/modules/expansion/btc_steroids.py +238 -0
  52. misp_modules/modules/expansion/censys_enrich.py +287 -0
  53. misp_modules/modules/expansion/circl_passivedns.py +85 -0
  54. misp_modules/modules/expansion/circl_passivessl.py +113 -0
  55. misp_modules/modules/expansion/clamav.py +129 -0
  56. misp_modules/modules/expansion/cluster25_expand.py +239 -0
  57. misp_modules/modules/expansion/countrycode.py +69 -0
  58. misp_modules/modules/expansion/cpe.py +140 -0
  59. misp_modules/modules/expansion/crowdsec.py +235 -0
  60. misp_modules/modules/expansion/crowdstrike_falcon.py +148 -0
  61. misp_modules/modules/expansion/cuckoo_submit.py +161 -0
  62. misp_modules/modules/expansion/cve.py +57 -0
  63. misp_modules/modules/expansion/cve_advanced.py +185 -0
  64. misp_modules/modules/expansion/cytomic_orion.py +196 -0
  65. misp_modules/modules/expansion/dbl_spamhaus.py +78 -0
  66. misp_modules/modules/expansion/dns.py +71 -0
  67. misp_modules/modules/expansion/docx_enrich.py +71 -0
  68. misp_modules/modules/expansion/domaintools.py +290 -0
  69. misp_modules/modules/expansion/eql.py +91 -0
  70. misp_modules/modules/expansion/eupi.py +86 -0
  71. misp_modules/modules/expansion/extract_url_components.py +83 -0
  72. misp_modules/modules/expansion/farsight_passivedns.py +243 -0
  73. misp_modules/modules/expansion/geoip_asn.py +75 -0
  74. misp_modules/modules/expansion/geoip_city.py +75 -0
  75. misp_modules/modules/expansion/geoip_country.py +73 -0
  76. misp_modules/modules/expansion/google_safe_browsing.py +86 -0
  77. misp_modules/modules/expansion/google_search.py +57 -0
  78. misp_modules/modules/expansion/google_threat_intelligence.py +453 -0
  79. misp_modules/modules/expansion/greynoise.py +346 -0
  80. misp_modules/modules/expansion/hashdd.py +53 -0
  81. misp_modules/modules/expansion/hashlookup.py +118 -0
  82. misp_modules/modules/expansion/hibp.py +62 -0
  83. misp_modules/modules/expansion/html_to_markdown.py +63 -0
  84. misp_modules/modules/expansion/hyasinsight.py +881 -0
  85. misp_modules/modules/expansion/intel471.py +73 -0
  86. misp_modules/modules/expansion/intelmq_eventdb.py.experimental +67 -0
  87. misp_modules/modules/expansion/ip2locationio.py +90 -0
  88. misp_modules/modules/expansion/ipasn.py +70 -0
  89. misp_modules/modules/expansion/ipinfo.py +112 -0
  90. misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py +633 -0
  91. misp_modules/modules/expansion/iprep.py +98 -0
  92. misp_modules/modules/expansion/jinja_template_rendering.py +54 -0
  93. misp_modules/modules/expansion/joesandbox_query.py +91 -0
  94. misp_modules/modules/expansion/joesandbox_submit.py +147 -0
  95. misp_modules/modules/expansion/lastline_query.py +148 -0
  96. misp_modules/modules/expansion/lastline_submit.py +180 -0
  97. misp_modules/modules/expansion/macaddress_io.py +130 -0
  98. misp_modules/modules/expansion/macvendors.py +55 -0
  99. misp_modules/modules/expansion/malshare_upload.py +107 -0
  100. misp_modules/modules/expansion/malwarebazaar.py +67 -0
  101. misp_modules/modules/expansion/mcafee_insights_enrich.py +249 -0
  102. misp_modules/modules/expansion/mmdb_lookup.py +138 -0
  103. misp_modules/modules/expansion/module.py.skeleton +44 -0
  104. misp_modules/modules/expansion/mwdb.py +152 -0
  105. misp_modules/modules/expansion/ocr_enrich.py +74 -0
  106. misp_modules/modules/expansion/ods_enrich.py +71 -0
  107. misp_modules/modules/expansion/odt_enrich.py +61 -0
  108. misp_modules/modules/expansion/onyphe.py +243 -0
  109. misp_modules/modules/expansion/onyphe_full.py +387 -0
  110. misp_modules/modules/expansion/otx.py +167 -0
  111. misp_modules/modules/expansion/passive_ssh.py +150 -0
  112. misp_modules/modules/expansion/passivetotal.py +358 -0
  113. misp_modules/modules/expansion/pdf_enrich.py +58 -0
  114. misp_modules/modules/expansion/pptx_enrich.py +65 -0
  115. misp_modules/modules/expansion/qintel_qsentry.py +229 -0
  116. misp_modules/modules/expansion/qrcode.py +99 -0
  117. misp_modules/modules/expansion/ransomcoindb.py +82 -0
  118. misp_modules/modules/expansion/rbl.py +126 -0
  119. misp_modules/modules/expansion/recordedfuture.py +544 -0
  120. misp_modules/modules/expansion/reversedns.py +77 -0
  121. misp_modules/modules/expansion/securitytrails.py +571 -0
  122. misp_modules/modules/expansion/shodan.py +244 -0
  123. misp_modules/modules/expansion/sigma_queries.py +61 -0
  124. misp_modules/modules/expansion/sigma_syntax_validator.py +49 -0
  125. misp_modules/modules/expansion/sigmf_expand.py +303 -0
  126. misp_modules/modules/expansion/socialscan.py +108 -0
  127. misp_modules/modules/expansion/sophoslabs_intelix.py +146 -0
  128. misp_modules/modules/expansion/sourcecache.py +57 -0
  129. misp_modules/modules/expansion/stairwell.py +156 -0
  130. misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +56 -0
  131. misp_modules/modules/expansion/threatcrowd.py +170 -0
  132. misp_modules/modules/expansion/threatfox.py +75 -0
  133. misp_modules/modules/expansion/threatminer.py +152 -0
  134. misp_modules/modules/expansion/triage_submit.py +119 -0
  135. misp_modules/modules/expansion/trustar_enrich.py +235 -0
  136. misp_modules/modules/expansion/urlhaus.py +164 -0
  137. misp_modules/modules/expansion/urlscan.py +269 -0
  138. misp_modules/modules/expansion/variotdbs.py +133 -0
  139. misp_modules/modules/expansion/virustotal.py +311 -0
  140. misp_modules/modules/expansion/virustotal_public.py +272 -0
  141. misp_modules/modules/expansion/virustotal_upload.py +91 -0
  142. misp_modules/modules/expansion/vmray_submit.py +161 -0
  143. misp_modules/modules/expansion/vmware_nsx.py +628 -0
  144. misp_modules/modules/expansion/vulndb.py +292 -0
  145. misp_modules/modules/expansion/vulnerability_lookup.py +324 -0
  146. misp_modules/modules/expansion/vulners.py +81 -0
  147. misp_modules/modules/expansion/vysion.py +221 -0
  148. misp_modules/modules/expansion/whois.py +72 -0
  149. misp_modules/modules/expansion/whoisfreaks.py +232 -0
  150. misp_modules/modules/expansion/wiki.py +58 -0
  151. misp_modules/modules/expansion/xforceexchange.py +189 -0
  152. misp_modules/modules/expansion/xlsx_enrich.py +63 -0
  153. misp_modules/modules/expansion/yara_query.py +70 -0
  154. misp_modules/modules/expansion/yara_syntax_validator.py +49 -0
  155. misp_modules/modules/expansion/yeti.py +196 -0
  156. misp_modules/modules/export_mod/__init__.py +3 -0
  157. misp_modules/modules/export_mod/cef_export.py +93 -0
  158. misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +150 -0
  159. misp_modules/modules/export_mod/defender_endpoint_export.py +141 -0
  160. misp_modules/modules/export_mod/goamlexport.py +255 -0
  161. misp_modules/modules/export_mod/liteexport.py +97 -0
  162. misp_modules/modules/export_mod/mass_eql_export.py +106 -0
  163. misp_modules/modules/export_mod/nexthinkexport.py +131 -0
  164. misp_modules/modules/export_mod/osqueryexport.py +125 -0
  165. misp_modules/modules/export_mod/pdfexport.py +107 -0
  166. misp_modules/modules/export_mod/testexport.py +73 -0
  167. misp_modules/modules/export_mod/threatStream_misp_export.py +114 -0
  168. misp_modules/modules/export_mod/threat_connect_export.py +125 -0
  169. misp_modules/modules/export_mod/virustotal_collections.py +141 -0
  170. misp_modules/modules/export_mod/vt_graph.py +120 -0
  171. misp_modules/modules/export_mod/yara_export.py +290 -0
  172. misp_modules/modules/import_mod/__init__.py +22 -0
  173. misp_modules/modules/import_mod/cof2misp.py +264 -0
  174. misp_modules/modules/import_mod/csvimport.py +319 -0
  175. misp_modules/modules/import_mod/cuckooimport.py +749 -0
  176. misp_modules/modules/import_mod/email_import.py +294 -0
  177. misp_modules/modules/import_mod/goamlimport.py +188 -0
  178. misp_modules/modules/import_mod/import_blueprint.py +93 -0
  179. misp_modules/modules/import_mod/joe_import.py +74 -0
  180. misp_modules/modules/import_mod/lastline_import.py +160 -0
  181. misp_modules/modules/import_mod/mispjson.py +73 -0
  182. misp_modules/modules/import_mod/ocr.py +122 -0
  183. misp_modules/modules/import_mod/openiocimport.py +99 -0
  184. misp_modules/modules/import_mod/taxii21.py +383 -0
  185. misp_modules/modules/import_mod/testimport.py +73 -0
  186. misp_modules/modules/import_mod/threatanalyzer_import.py +552 -0
  187. misp_modules/modules/import_mod/url_import.py +97 -0
  188. misp_modules/modules/import_mod/vmray_import.py +96 -0
  189. misp_modules/modules/import_mod/vmray_summary_json_import.py +87 -0
  190. misp_modules-2.4.196.dist-info/LICENSE +661 -0
  191. misp_modules-2.4.196.dist-info/METADATA +281 -0
  192. misp_modules-2.4.196.dist-info/RECORD +194 -0
  193. misp_modules-2.4.196.dist-info/WHEEL +4 -0
  194. misp_modules-2.4.196.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,244 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ import json
4
+ try:
5
+ import shodan
6
+ except ImportError:
7
+ print("shodan module not installed.")
8
+ from . import check_input_attribute, standard_error_message
9
+ from datetime import datetime
10
+ from pymisp import MISPAttribute, MISPEvent, MISPObject
11
+
12
+ misperrors = {'error': 'Error'}
13
+ mispattributes = {'input': ['ip-src', 'ip-dst'],
14
+ 'format': 'misp_standard'}
15
+ moduleinfo = {
16
+ 'version': '0.2',
17
+ 'author': 'Raphaël Vinot',
18
+ 'description': 'Module to query on Shodan.',
19
+ 'module-type': ['expansion'],
20
+ 'name': 'Shodan Lookup',
21
+ 'logo': 'shodan.png',
22
+ 'requirements': ['shodan python library', 'An access to the Shodan API (apikey)'],
23
+ 'features': 'The module takes an IP address as input and queries the Shodan API to get some additional data about it.',
24
+ 'references': ['https://www.shodan.io/'],
25
+ 'input': 'An IP address MISP attribute.',
26
+ 'output': 'Text with additional data about the input, resulting from the query on Shodan.',
27
+ }
28
+
29
+ moduleconfig = ['apikey']
30
+
31
+
32
+ class ShodanParser():
33
+ def __init__(self, attribute):
34
+ self.misp_event = MISPEvent()
35
+ self.attribute = MISPAttribute()
36
+ self.attribute.from_dict(**attribute)
37
+ self.misp_event.add_attribute(**self.attribute)
38
+ self.ip_address_mapping = {
39
+ 'asn': {'type': 'AS', 'object_relation': 'asn'},
40
+ 'city': {'type': 'text', 'object_relation': 'city'},
41
+ 'country_code': {'type': 'text', 'object_relation': 'country-code'},
42
+ 'country_name': {'type': 'text', 'object_relation': 'country'},
43
+ 'isp': {'type': 'text', 'object_relation': 'ISP'},
44
+ 'latitude': {'type': 'float', 'object_relation': 'latitude'},
45
+ 'longitude': {'type': 'float', 'object_relation': 'longitude'},
46
+ 'org': {'type': 'text', 'object_relation': 'organization'},
47
+ 'postal_code': {'type': 'text', 'object_relation': 'zipcode'},
48
+ 'region_code': {'type': 'text', 'object_relation': 'region-code'}
49
+ }
50
+ self.ip_port_mapping = {
51
+ 'domains': {'type': 'domain', 'object_relation': 'domain'},
52
+ 'hostnames': {'type': 'hostname', 'object_relation': 'hostname'}
53
+ }
54
+ self.vulnerability_mapping = {
55
+ 'cvss': {'type': 'float', 'object_relation': 'cvss-score'},
56
+ 'summary': {'type': 'text', 'object_relation': 'summary'}
57
+ }
58
+ self.x509_mapping = {
59
+ 'bits': {'type': 'text', 'object_relation': 'pubkey-info-size'},
60
+ 'expires': {'type': 'datetime', 'object_relation': 'validity-not-after'},
61
+ 'issued': {'type': 'datetime', 'object_relation': 'validity-not-before'},
62
+ 'issuer': {'type': 'text', 'object_relation': 'issuer'},
63
+ 'serial': {'type': 'text', 'object_relation': 'serial-number'},
64
+ 'sig_alg': {'type': 'text', 'object_relation': 'signature_algorithm'},
65
+ 'subject': {'type': 'text', 'object_relation': 'subject'},
66
+ 'type': {'type': 'text', 'object_relation': 'pubkey-info-algorithm'},
67
+ 'version': {'type': 'text', 'object_relation': 'version'}
68
+ }
69
+
70
+ def query_shodan(self, apikey):
71
+ # Query Shodan and get the results in a json blob
72
+ api = shodan.Shodan(apikey)
73
+ query_results = api.host(self.attribute.value)
74
+
75
+ # Parse the information about the IP address used as input
76
+ ip_address_attributes = []
77
+ for feature, mapping in self.ip_address_mapping.items():
78
+ if query_results.get(feature):
79
+ attribute = {'value': query_results[feature]}
80
+ attribute.update(mapping)
81
+ ip_address_attributes.append(attribute)
82
+ if ip_address_attributes:
83
+ ip_address_object = MISPObject('ip-api-address')
84
+ for attribute in ip_address_attributes:
85
+ ip_address_object.add_attribute(**attribute)
86
+ ip_address_object.add_reference(self.attribute.uuid, 'describes')
87
+ self.misp_event.add_object(ip_address_object)
88
+
89
+ # Parse the hostnames / domains and ports associated with the IP address
90
+ if query_results.get('ports'):
91
+ ip_port_object = MISPObject('ip-port')
92
+ ip_port_object.add_attribute(**self._get_source_attribute())
93
+ feature = self.attribute.type.split('-')[1]
94
+ for port in query_results['ports']:
95
+ attribute = {
96
+ 'type': 'port',
97
+ 'object_relation': f'{feature}-port',
98
+ 'value': port
99
+ }
100
+ ip_port_object.add_attribute(**attribute)
101
+ for feature, mapping in self.ip_port_mapping.items():
102
+ for value in query_results.get(feature, []):
103
+ attribute = {'value': value}
104
+ attribute.update(mapping)
105
+ ip_port_object.add_attribute(**attribute)
106
+ ip_port_object.add_reference(self.attribute.uuid, 'extends')
107
+ self.misp_event.add_object(ip_port_object)
108
+ else:
109
+ if any(query_results.get(feature) for feature in ('domains', 'hostnames')):
110
+ domain_ip_object = MISPObject('domain-ip')
111
+ domain_ip_object.add_attribute(**self._get_source_attribute())
112
+ for feature in ('domains', 'hostnames'):
113
+ for value in query_results[feature]:
114
+ attribute = {
115
+ 'type': 'domain',
116
+ 'object_relation': 'domain',
117
+ 'value': value
118
+ }
119
+ domain_ip_object.add_attribute(**attribute)
120
+ domain_ip_object.add_reference(self.attribute.uuid, 'extends')
121
+ self.misp_event.add_object(domain_ip_object)
122
+
123
+ # Parse data within the "data" field
124
+ if query_results.get('vulns'):
125
+ vulnerabilities = {}
126
+ for data in query_results['data']:
127
+ # Parse vulnerabilities
128
+ if data.get('vulns'):
129
+ for cve, vulnerability in data['vulns'].items():
130
+ if cve not in vulnerabilities:
131
+ vulnerabilities[cve] = vulnerability
132
+ # Also parse the certificates
133
+ if data.get('ssl'):
134
+ self._parse_cert(data['ssl'])
135
+ for cve, vulnerability in vulnerabilities.items():
136
+ vulnerability_object = MISPObject('vulnerability')
137
+ vulnerability_object.add_attribute(**{
138
+ 'type': 'vulnerability',
139
+ 'object_relation': 'id',
140
+ 'value': cve
141
+ })
142
+ for feature, mapping in self.vulnerability_mapping.items():
143
+ if vulnerability.get(feature):
144
+ attribute = {'value': vulnerability[feature]}
145
+ attribute.update(mapping)
146
+ vulnerability_object.add_attribute(**attribute)
147
+ if vulnerability.get('references'):
148
+ for reference in vulnerability['references']:
149
+ vulnerability_object.add_attribute(**{
150
+ 'type': 'link',
151
+ 'object_relation': 'references',
152
+ 'value': reference
153
+ })
154
+ vulnerability_object.add_reference(self.attribute.uuid, 'vulnerability-of')
155
+ self.misp_event.add_object(vulnerability_object)
156
+ for cve_id in query_results['vulns']:
157
+ if cve_id not in vulnerabilities:
158
+ attribute = {
159
+ 'type': 'vulnerability',
160
+ 'value': cve_id
161
+ }
162
+ self.misp_event.add_attribute(**attribute)
163
+ else:
164
+ # We have no vulnerability data, we only check if we have
165
+ # certificates within the "data" field
166
+ for data in query_results['data']:
167
+ if data.get('ssl'):
168
+ self._parse_cert(data['ssl']['cert'])
169
+
170
+ def get_result(self):
171
+ event = json.loads(self.misp_event.to_json())
172
+ results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
173
+ return {'results': results}
174
+
175
+ # When we want to add the IP address information in objects such as the
176
+ # domain-ip or ip-port objects referencing the input IP address attribute
177
+ def _get_source_attribute(self):
178
+ return {
179
+ 'type': self.attribute.type,
180
+ 'object_relation': self.attribute.type,
181
+ 'value': self.attribute.value
182
+ }
183
+
184
+ def _parse_cert(self, certificate):
185
+ x509_object = MISPObject('x509')
186
+ for feature in ('serial', 'sig_alg', 'version'):
187
+ if certificate.get(feature):
188
+ attribute = {'value': certificate[feature]}
189
+ attribute.update(self.x509_mapping[feature])
190
+ x509_object.add_attribute(**attribute)
191
+ # Parse issuer and subject value
192
+ for feature in ('issuer', 'subject'):
193
+ if certificate.get(feature):
194
+ attribute_value = (f'{identifier}={value}' for identifier, value in certificate[feature].items())
195
+ attribute = {'value': f'/{"/".join(attribute_value)}'}
196
+ attribute.update(self.x509_mapping[feature])
197
+ x509_object.add_attribute(**attribute)
198
+ # Parse datetime attributes
199
+ for feature in ('expires', 'issued'):
200
+ if certificate.get(feature):
201
+ attribute = {'value': datetime.strptime(certificate[feature], '%Y%m%d%H%M%SZ')}
202
+ attribute.update(self.x509_mapping[feature])
203
+ x509_object.add_attribute(**attribute)
204
+ # Parse fingerprints
205
+ if certificate.get('fingerprint'):
206
+ for hash_type, hash_value in certificate['fingerprint'].items():
207
+ x509_object.add_attribute(**{
208
+ 'type': f'x509-fingerprint-{hash_type}',
209
+ 'object_relation': f'x509-fingerprint-{hash_type}',
210
+ 'value': hash_value
211
+ })
212
+ # Parse public key related info
213
+ if certificate.get('pubkey'):
214
+ for feature, value in certificate['pubkey'].items():
215
+ attribute = {'value': value}
216
+ attribute.update(self.x509_mapping[feature])
217
+ x509_object.add_attribute(**attribute)
218
+ x509_object.add_reference(self.attribute.uuid, 'identifies')
219
+ self.misp_event.add_object(x509_object)
220
+
221
+
222
+ def handler(q=False):
223
+ if q is False:
224
+ return False
225
+ request = json.loads(q)
226
+ if not request.get('config', {}).get('apikey'):
227
+ return {'error': 'Shodan authentication is missing'}
228
+ if not request.get('attribute') or not check_input_attribute(request['attribute']):
229
+ return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
230
+ attribute = request['attribute']
231
+ if attribute['type'] not in mispattributes['input']:
232
+ return {'error': 'Unsupported attribute type.'}
233
+ shodan_parser = ShodanParser(attribute)
234
+ shodan_parser.query_shodan(request['config']['apikey'])
235
+ return shodan_parser.get_result()
236
+
237
+
238
+ def introspection():
239
+ return mispattributes
240
+
241
+
242
+ def version():
243
+ moduleinfo['config'] = moduleconfig
244
+ return moduleinfo
@@ -0,0 +1,61 @@
1
+ import io
2
+ import json
3
+ try:
4
+ from sigma.parser.collection import SigmaCollectionParser
5
+ from sigma.configuration import SigmaConfiguration
6
+ from sigma.backends.discovery import getBackend
7
+ except ImportError:
8
+ print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")
9
+
10
+ misperrors = {'error': 'Error'}
11
+ mispattributes = {'input': ['sigma'], 'output': ['text']}
12
+ moduleinfo = {
13
+ 'version': '0.1',
14
+ 'author': 'Christian Studer',
15
+ 'module-type': ['expansion', 'hover'],
16
+ 'name': 'Sigma Rule Converter',
17
+ 'description': 'An expansion hover module to display the result of sigma queries.',
18
+ 'logo': 'sigma.png',
19
+ 'requirements': ['Sigma python library'],
20
+ 'features': 'This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs.',
21
+ 'references': ['https://github.com/Neo23x0/sigma/wiki'],
22
+ 'input': 'A Sigma attribute.',
23
+ 'output': 'Text displaying results of queries on the Sigma attribute.',
24
+ }
25
+ moduleconfig = []
26
+ sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'mdatp', 'splunkxml', 'arcsight', 'qualys')
27
+
28
+
29
+ def handler(q=False):
30
+ if q is False:
31
+ return False
32
+ request = json.loads(q)
33
+ if not request.get('sigma'):
34
+ misperrors['error'] = 'Sigma rule missing'
35
+ return misperrors
36
+ config = SigmaConfiguration()
37
+ f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8')
38
+ parser = SigmaCollectionParser(f, config)
39
+ targets = []
40
+ results = []
41
+ for t in sigma_targets:
42
+ backend = getBackend(t)(config, {'rulecomment': False})
43
+ try:
44
+ parser.generate(backend)
45
+ result = backend.finalize()
46
+ if result:
47
+ results.append(result)
48
+ targets.append(t)
49
+ except Exception:
50
+ continue
51
+ d_result = {t: r.strip() for t, r in zip(targets, results)}
52
+ return {'results': [{'types': mispattributes['output'], 'values': d_result}]}
53
+
54
+
55
+ def introspection():
56
+ return mispattributes
57
+
58
+
59
+ def version():
60
+ moduleinfo['config'] = moduleconfig
61
+ return moduleinfo
@@ -0,0 +1,49 @@
1
+ import json
2
+ try:
3
+ import yaml
4
+ from sigma.parser.rule import SigmaParser
5
+ from sigma.configuration import SigmaConfiguration
6
+ except ImportError:
7
+ print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")
8
+
9
+ misperrors = {'error': 'Error'}
10
+ mispattributes = {'input': ['sigma'], 'output': ['text']}
11
+ moduleinfo = {
12
+ 'version': '0.1',
13
+ 'author': 'Christian Studer',
14
+ 'module-type': ['expansion', 'hover'],
15
+ 'name': 'Sigma Syntax Validator',
16
+ 'description': 'An expansion hover module to perform a syntax check on sigma rules.',
17
+ 'logo': 'sigma.png',
18
+ 'requirements': ['Sigma python library', 'Yaml python library'],
19
+ 'features': 'This module takes a Sigma rule attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise.',
20
+ 'references': ['https://github.com/Neo23x0/sigma/wiki'],
21
+ 'input': 'A Sigma attribute.',
22
+ 'output': 'Text describing the validity of the Sigma rule.',
23
+ }
24
+ moduleconfig = []
25
+
26
+
27
+ def handler(q=False):
28
+ if q is False:
29
+ return False
30
+ request = json.loads(q)
31
+ if not request.get('sigma'):
32
+ misperrors['error'] = 'Sigma rule missing'
33
+ return misperrors
34
+ config = SigmaConfiguration()
35
+ try:
36
+ parser = SigmaParser(yaml.safe_load(request.get('sigma')), config)
37
+ result = ("Syntax valid: {}".format(parser.values))
38
+ except Exception as e:
39
+ result = ("Syntax error: {}".format(str(e)))
40
+ return {'results': [{'types': mispattributes['output'], 'values': result}]}
41
+
42
+
43
+ def introspection():
44
+ return mispattributes
45
+
46
+
47
+ def version():
48
+ moduleinfo['config'] = moduleconfig
49
+ return moduleinfo
@@ -0,0 +1,303 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ import base64
4
+ import numpy as np
5
+ import matplotlib.pyplot as plt
6
+ import io
7
+ import json
8
+ import tempfile
9
+ import logging
10
+ import sys
11
+ from pymisp import MISPObject, MISPEvent
12
+ from sigmf import SigMFFile
13
+ from sigmf.archive import SIGMF_DATASET_EXT, SIGMF_METADATA_EXT
14
+ import tarfile
15
+
16
+ log = logging.getLogger("sigmf-expand")
17
+ log.setLevel(logging.DEBUG)
18
+ sh = logging.StreamHandler(sys.stdout)
19
+ sh.setLevel(logging.DEBUG)
20
+ fmt = logging.Formatter(
21
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
22
+ )
23
+ sh.setFormatter(fmt)
24
+ log.addHandler(sh)
25
+
26
+ misperrors = {'error': 'Error'}
27
+ mispattributes = {'input': ['sigmf-recording', 'sigmf-archive'], 'output': [
28
+ 'MISP objects'], 'format': 'misp_standard'}
29
+ moduleinfo = {
30
+ 'version': '0.1',
31
+ 'author': 'Luciano Righetti',
32
+ 'description': 'Expands a SigMF Recording object into a SigMF Expanded Recording object, extracts a SigMF archive into a SigMF Recording object.',
33
+ 'module-type': ['expansion'],
34
+ 'name': 'SigMF Expansion',
35
+ 'logo': '',
36
+ 'requirements': [],
37
+ 'features': '',
38
+ 'references': [],
39
+ 'input': '',
40
+ 'output': '',
41
+ }
42
+
43
+
44
+ def get_samples(data_bytes, data_type) -> np.ndarray:
45
+ """
46
+ Get samples from bytes.
47
+
48
+ Source: https://github.com/IQEngine/IQEngine/blob/main/api/rf/samples.py
49
+
50
+ Parameters
51
+ ----------
52
+ data_bytes : bytes
53
+ The bytes to convert to samples.
54
+ data_type : str
55
+ The data type of the bytes.
56
+
57
+ Returns
58
+ -------
59
+ np.ndarray
60
+ The samples.
61
+ """
62
+
63
+ if data_type == "ci16_le" or data_type == "ci16":
64
+ samples = np.frombuffer(data_bytes, dtype=np.int16)
65
+ samples = samples[::2] + 1j * samples[1::2]
66
+ elif data_type == "cf32_le":
67
+ samples = np.frombuffer(data_bytes, dtype=np.complex64)
68
+ elif data_type == "ci8" or data_type == "i8":
69
+ samples = np.frombuffer(data_bytes, dtype=np.int8)
70
+ samples = samples[::2] + 1j * samples[1::2]
71
+ else:
72
+ raise ("Datatype " + data_type + " not implemented")
73
+ return samples
74
+
75
+
76
+ def generate_plots(recording, meta_filename, data_bytes):
77
+ # FFT plot
78
+ filename = meta_filename.replace('.sigmf-data', '')
79
+ samples = get_samples(
80
+ data_bytes, recording.get_global_info()['core:datatype'])
81
+ sample_rate = recording.get_global_info()['core:sample_rate']
82
+
83
+ # Waterfall plot
84
+ # snippet from https://pysdr.org/content/frequency_domain.html#fast-fourier-transform-fft
85
+ fft_size = 1024
86
+ # // is an integer division which rounds down
87
+ num_rows = len(samples) // fft_size
88
+ spectrogram = np.zeros((num_rows, fft_size))
89
+ for i in range(num_rows):
90
+ spectrogram[i, :] = 10 * \
91
+ np.log10(np.abs(np.fft.fftshift(
92
+ np.fft.fft(samples[i*fft_size:(i+1)*fft_size])))**2)
93
+
94
+ plt.figure(figsize=(10, 4))
95
+ plt.title(filename)
96
+ plt.imshow(spectrogram, aspect='auto', extent=[
97
+ sample_rate/-2/1e6, sample_rate/2/1e6, 0, len(samples)/sample_rate])
98
+ plt.xlabel("Frequency [MHz]")
99
+ plt.ylabel("Time [ms]")
100
+ plt.savefig(filename + '-spectrogram.png')
101
+ waterfall_buff = io.BytesIO()
102
+ plt.savefig(waterfall_buff, format='png')
103
+ waterfall_buff.seek(0)
104
+ waterfall_png = base64.b64encode(waterfall_buff.read()).decode('utf-8')
105
+
106
+ waterfall_attr = {
107
+ 'type': 'attachment',
108
+ 'value': filename + '-waterfall.png',
109
+ 'data': waterfall_png,
110
+ 'comment': 'Waterfall plot of the recording'
111
+ }
112
+
113
+ return [{'relation': 'waterfall-plot', 'attribute': waterfall_attr}]
114
+
115
+
116
+ def process_sigmf_archive(object):
117
+
118
+ event = MISPEvent()
119
+ sigmf_data_attr = None
120
+ sigmf_meta_attr = None
121
+
122
+ try:
123
+ # get sigmf-archive attribute
124
+ for attribute in object['Attribute']:
125
+ if attribute['object_relation'] == 'SigMF-archive':
126
+
127
+ # write temp data file to disk
128
+ sigmf_archive_file = tempfile.NamedTemporaryFile(
129
+ suffix='.sigmf')
130
+ sigmf_archive_bin = base64.b64decode(attribute['data'])
131
+ with open(sigmf_archive_file.name, 'wb') as f:
132
+ f.write(sigmf_archive_bin)
133
+ f.close()
134
+
135
+ sigmf_tarfile = tarfile.open(
136
+ sigmf_archive_file.name, mode="r", format=tarfile.PAX_FORMAT)
137
+
138
+ files = sigmf_tarfile.getmembers()
139
+
140
+ for file in files:
141
+ if file.name.endswith(SIGMF_METADATA_EXT):
142
+ metadata_reader = sigmf_tarfile.extractfile(file)
143
+ sigmf_meta_attr = {
144
+ 'type': 'attachment',
145
+ 'value': file.name,
146
+ 'data': base64.b64encode(metadata_reader.read()).decode("utf-8"),
147
+ 'comment': 'SigMF metadata file',
148
+ 'object_relation': 'SigMF-meta'
149
+ }
150
+
151
+ if file.name.endswith(SIGMF_DATASET_EXT):
152
+ data_reader = sigmf_tarfile.extractfile(file)
153
+ sigmf_data_attr = {
154
+ 'type': 'attachment',
155
+ 'value': file.name,
156
+ 'data': base64.b64encode(data_reader.read()).decode("utf-8"),
157
+ 'comment': 'SigMF data file',
158
+ 'object_relation': 'SigMF-data'
159
+ }
160
+
161
+ if sigmf_meta_attr is None:
162
+ return {"error": "No SigMF metadata file found"}
163
+
164
+ recording = MISPObject('sigmf-recording')
165
+ recording.add_attribute(**sigmf_meta_attr)
166
+ recording.add_attribute(**sigmf_data_attr)
167
+
168
+ # add reference to original SigMF Archive object
169
+ recording.add_reference(object['uuid'], "expands")
170
+
171
+ event.add_object(recording)
172
+ event = json.loads(event.to_json())
173
+
174
+ return {"results": {'Object': event['Object']}}
175
+
176
+ # no sigmf-archive attribute found
177
+ return {"error": "No SigMF-archive attribute found"}
178
+
179
+ except Exception as e:
180
+ logging.exception(e)
181
+ return {"error": "An error occured when processing the SigMF archive"}
182
+
183
+
184
+ def process_sigmf_recording(object):
185
+
186
+ event = MISPEvent()
187
+
188
+ for attribute in object['Attribute']:
189
+ if attribute['object_relation'] == 'SigMF-data':
190
+ sigmf_data_attr = attribute
191
+
192
+ if attribute['object_relation'] == 'SigMF-meta':
193
+ sigmf_meta_attr = attribute
194
+
195
+ if sigmf_meta_attr is None:
196
+ return {"error": "No SigMF-data attribute"}
197
+
198
+ if sigmf_data_attr is None:
199
+ return {"error": "No SigMF-meta attribute"}
200
+
201
+ try:
202
+ sigmf_meta = base64.b64decode(sigmf_meta_attr['data']).decode('utf-8')
203
+ sigmf_meta = json.loads(sigmf_meta)
204
+ except Exception as e:
205
+ logging.exception(e)
206
+ return {"error": "Provided .sigmf-meta is not a valid JSON string"}
207
+
208
+ # write temp data file to disk
209
+ sigmf_data_file = tempfile.NamedTemporaryFile(suffix='.sigmf-data')
210
+ sigmf_data_bin = base64.b64decode(sigmf_data_attr['data'])
211
+ with open(sigmf_data_file.name, 'wb') as f:
212
+ f.write(sigmf_data_bin)
213
+ f.close()
214
+
215
+ try:
216
+ recording = SigMFFile(
217
+ metadata=sigmf_meta,
218
+ data_file=sigmf_data_file.name
219
+ )
220
+ except Exception as e:
221
+ logging.exception(e)
222
+ return {"error": "Provided .sigmf-meta and .sigmf-data is not a valid SigMF file"}
223
+
224
+ expanded_sigmf = MISPObject('sigmf-expanded-recording')
225
+
226
+ if 'core:author' in sigmf_meta['global']:
227
+ expanded_sigmf.add_attribute(
228
+ 'author', **{'type': 'text', 'value': sigmf_meta['global']['core:author']})
229
+ if 'core:datatype' in sigmf_meta['global']:
230
+ expanded_sigmf.add_attribute(
231
+ 'datatype', **{'type': 'text', 'value': sigmf_meta['global']['core:datatype']})
232
+ if 'core:description' in sigmf_meta['global']:
233
+ expanded_sigmf.add_attribute(
234
+ 'description', **{'type': 'text', 'value': sigmf_meta['global']['core:description']})
235
+ if 'core:license' in sigmf_meta['global']:
236
+ expanded_sigmf.add_attribute(
237
+ 'license', **{'type': 'text', 'value': sigmf_meta['global']['core:license']})
238
+ if 'core:num_channels' in sigmf_meta['global']:
239
+ expanded_sigmf.add_attribute(
240
+ 'num_channels', **{'type': 'counter', 'value': sigmf_meta['global']['core:num_channels']})
241
+ if 'core:recorder' in sigmf_meta['global']:
242
+ expanded_sigmf.add_attribute(
243
+ 'recorder', **{'type': 'text', 'value': sigmf_meta['global']['core:recorder']})
244
+ if 'core:sample_rate' in sigmf_meta['global']:
245
+ expanded_sigmf.add_attribute(
246
+ 'sample_rate', **{'type': 'float', 'value': sigmf_meta['global']['core:sample_rate']})
247
+ if 'core:sha512' in sigmf_meta['global']:
248
+ expanded_sigmf.add_attribute(
249
+ 'sha512', **{'type': 'text', 'value': sigmf_meta['global']['core:sha512']})
250
+ if 'core:version' in sigmf_meta['global']:
251
+ expanded_sigmf.add_attribute(
252
+ 'version', **{'type': 'text', 'value': sigmf_meta['global']['core:version']})
253
+
254
+ # add reference to original SigMF Recording object
255
+ expanded_sigmf.add_reference(object['uuid'], "expands")
256
+
257
+ # add FFT and waterfall plot
258
+ try:
259
+ plots = generate_plots(
260
+ recording, sigmf_data_attr['value'], sigmf_data_bin)
261
+ except Exception as e:
262
+ logging.exception(e)
263
+ return {"error": "Could not generate plots"}
264
+
265
+ for plot in plots:
266
+ expanded_sigmf.add_attribute(plot['relation'], **plot['attribute'])
267
+
268
+ event.add_object(expanded_sigmf)
269
+ event = json.loads(event.to_json())
270
+
271
+ return {"results": {'Object': event['Object']}}
272
+
273
+
274
+ def handler(q=False):
275
+ request = json.loads(q)
276
+ object = request.get("object")
277
+ event = MISPEvent()
278
+
279
+ if not object:
280
+ return {"error": "No object provided"}
281
+
282
+ if 'Attribute' not in object:
283
+ return {"error": "Empty Attribute list"}
284
+
285
+ # check if it's a SigMF Archive
286
+ if object['name'] == 'sigmf-archive':
287
+ return process_sigmf_archive(object)
288
+
289
+ # check if it's a SigMF Recording
290
+ if object['name'] == 'sigmf-recording':
291
+ return process_sigmf_recording(object)
292
+
293
+ # TODO: add support for SigMF Collection
294
+
295
+ return {"error": "No SigMF object provided"}
296
+
297
+
298
+ def introspection():
299
+ return mispattributes
300
+
301
+
302
+ def version():
303
+ return moduleinfo