misp-modules 2.4.196__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. misp_modules/__init__.py +379 -0
  2. misp_modules/helpers/__init__.py +1 -0
  3. misp_modules/helpers/cache.py +84 -0
  4. misp_modules/lib/__init__.py +4 -0
  5. misp_modules/lib/_vmray/__init__.py +0 -0
  6. misp_modules/lib/_vmray/parser.py +1417 -0
  7. misp_modules/lib/_vmray/rest_api.py +148 -0
  8. misp_modules/lib/cof2misp/LICENSE-2.0.txt +202 -0
  9. misp_modules/lib/cof2misp/__init__.py +0 -0
  10. misp_modules/lib/cof2misp/cof.py +165 -0
  11. misp_modules/lib/joe_mapping.py +114 -0
  12. misp_modules/lib/joe_parser.py +573 -0
  13. misp_modules/lib/lastline_api.py +841 -0
  14. misp_modules/lib/qintel_helper.py +263 -0
  15. misp_modules/lib/stix2misp.py +2080 -0
  16. misp_modules/lib/stix2misp_mapping.py +460 -0
  17. misp_modules/lib/synonymsToTagNames.json +1 -0
  18. misp_modules/lib/vt_graph_parser/__init__.py +8 -0
  19. misp_modules/lib/vt_graph_parser/errors.py +20 -0
  20. misp_modules/lib/vt_graph_parser/helpers/__init__.py +7 -0
  21. misp_modules/lib/vt_graph_parser/helpers/parsers.py +88 -0
  22. misp_modules/lib/vt_graph_parser/helpers/rules.py +304 -0
  23. misp_modules/lib/vt_graph_parser/helpers/wrappers.py +58 -0
  24. misp_modules/lib/vt_graph_parser/importers/__init__.py +7 -0
  25. misp_modules/lib/vt_graph_parser/importers/base.py +98 -0
  26. misp_modules/lib/vt_graph_parser/importers/pymisp_response.py +73 -0
  27. misp_modules/modules/__init__.py +4 -0
  28. misp_modules/modules/action_mod/__init__.py +1 -0
  29. misp_modules/modules/action_mod/_utils/__init__.py +1 -0
  30. misp_modules/modules/action_mod/_utils/utils.py +70 -0
  31. misp_modules/modules/action_mod/mattermost.py +113 -0
  32. misp_modules/modules/action_mod/slack.py +96 -0
  33. misp_modules/modules/action_mod/testaction.py +68 -0
  34. misp_modules/modules/expansion/__init__.py +36 -0
  35. misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT +27 -0
  36. misp_modules/modules/expansion/_dnsdb_query/LICENSE +202 -0
  37. misp_modules/modules/expansion/_dnsdb_query/README.md +162 -0
  38. misp_modules/modules/expansion/_dnsdb_query/__init__.py +0 -0
  39. misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +327 -0
  40. misp_modules/modules/expansion/_ransomcoindb/__init__.py +0 -0
  41. misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py +96 -0
  42. misp_modules/modules/expansion/_vulnerability_parser/__init__.py +0 -0
  43. misp_modules/modules/expansion/_vulnerability_parser/vulnerability_parser.py +118 -0
  44. misp_modules/modules/expansion/abuseipdb.py +146 -0
  45. misp_modules/modules/expansion/apiosintds.py +365 -0
  46. misp_modules/modules/expansion/apivoid.py +133 -0
  47. misp_modules/modules/expansion/assemblyline_query.py +179 -0
  48. misp_modules/modules/expansion/assemblyline_submit.py +100 -0
  49. misp_modules/modules/expansion/backscatter_io.py +84 -0
  50. misp_modules/modules/expansion/btc_scam_check.py +54 -0
  51. misp_modules/modules/expansion/btc_steroids.py +238 -0
  52. misp_modules/modules/expansion/censys_enrich.py +287 -0
  53. misp_modules/modules/expansion/circl_passivedns.py +85 -0
  54. misp_modules/modules/expansion/circl_passivessl.py +113 -0
  55. misp_modules/modules/expansion/clamav.py +129 -0
  56. misp_modules/modules/expansion/cluster25_expand.py +239 -0
  57. misp_modules/modules/expansion/countrycode.py +69 -0
  58. misp_modules/modules/expansion/cpe.py +140 -0
  59. misp_modules/modules/expansion/crowdsec.py +235 -0
  60. misp_modules/modules/expansion/crowdstrike_falcon.py +148 -0
  61. misp_modules/modules/expansion/cuckoo_submit.py +161 -0
  62. misp_modules/modules/expansion/cve.py +57 -0
  63. misp_modules/modules/expansion/cve_advanced.py +185 -0
  64. misp_modules/modules/expansion/cytomic_orion.py +196 -0
  65. misp_modules/modules/expansion/dbl_spamhaus.py +78 -0
  66. misp_modules/modules/expansion/dns.py +71 -0
  67. misp_modules/modules/expansion/docx_enrich.py +71 -0
  68. misp_modules/modules/expansion/domaintools.py +290 -0
  69. misp_modules/modules/expansion/eql.py +91 -0
  70. misp_modules/modules/expansion/eupi.py +86 -0
  71. misp_modules/modules/expansion/extract_url_components.py +83 -0
  72. misp_modules/modules/expansion/farsight_passivedns.py +243 -0
  73. misp_modules/modules/expansion/geoip_asn.py +75 -0
  74. misp_modules/modules/expansion/geoip_city.py +75 -0
  75. misp_modules/modules/expansion/geoip_country.py +73 -0
  76. misp_modules/modules/expansion/google_safe_browsing.py +86 -0
  77. misp_modules/modules/expansion/google_search.py +57 -0
  78. misp_modules/modules/expansion/google_threat_intelligence.py +453 -0
  79. misp_modules/modules/expansion/greynoise.py +346 -0
  80. misp_modules/modules/expansion/hashdd.py +53 -0
  81. misp_modules/modules/expansion/hashlookup.py +118 -0
  82. misp_modules/modules/expansion/hibp.py +62 -0
  83. misp_modules/modules/expansion/html_to_markdown.py +63 -0
  84. misp_modules/modules/expansion/hyasinsight.py +881 -0
  85. misp_modules/modules/expansion/intel471.py +73 -0
  86. misp_modules/modules/expansion/intelmq_eventdb.py.experimental +67 -0
  87. misp_modules/modules/expansion/ip2locationio.py +90 -0
  88. misp_modules/modules/expansion/ipasn.py +70 -0
  89. misp_modules/modules/expansion/ipinfo.py +112 -0
  90. misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py +633 -0
  91. misp_modules/modules/expansion/iprep.py +98 -0
  92. misp_modules/modules/expansion/jinja_template_rendering.py +54 -0
  93. misp_modules/modules/expansion/joesandbox_query.py +91 -0
  94. misp_modules/modules/expansion/joesandbox_submit.py +147 -0
  95. misp_modules/modules/expansion/lastline_query.py +148 -0
  96. misp_modules/modules/expansion/lastline_submit.py +180 -0
  97. misp_modules/modules/expansion/macaddress_io.py +130 -0
  98. misp_modules/modules/expansion/macvendors.py +55 -0
  99. misp_modules/modules/expansion/malshare_upload.py +107 -0
  100. misp_modules/modules/expansion/malwarebazaar.py +67 -0
  101. misp_modules/modules/expansion/mcafee_insights_enrich.py +249 -0
  102. misp_modules/modules/expansion/mmdb_lookup.py +138 -0
  103. misp_modules/modules/expansion/module.py.skeleton +44 -0
  104. misp_modules/modules/expansion/mwdb.py +152 -0
  105. misp_modules/modules/expansion/ocr_enrich.py +74 -0
  106. misp_modules/modules/expansion/ods_enrich.py +71 -0
  107. misp_modules/modules/expansion/odt_enrich.py +61 -0
  108. misp_modules/modules/expansion/onyphe.py +243 -0
  109. misp_modules/modules/expansion/onyphe_full.py +387 -0
  110. misp_modules/modules/expansion/otx.py +167 -0
  111. misp_modules/modules/expansion/passive_ssh.py +150 -0
  112. misp_modules/modules/expansion/passivetotal.py +358 -0
  113. misp_modules/modules/expansion/pdf_enrich.py +58 -0
  114. misp_modules/modules/expansion/pptx_enrich.py +65 -0
  115. misp_modules/modules/expansion/qintel_qsentry.py +229 -0
  116. misp_modules/modules/expansion/qrcode.py +99 -0
  117. misp_modules/modules/expansion/ransomcoindb.py +82 -0
  118. misp_modules/modules/expansion/rbl.py +126 -0
  119. misp_modules/modules/expansion/recordedfuture.py +544 -0
  120. misp_modules/modules/expansion/reversedns.py +77 -0
  121. misp_modules/modules/expansion/securitytrails.py +571 -0
  122. misp_modules/modules/expansion/shodan.py +244 -0
  123. misp_modules/modules/expansion/sigma_queries.py +61 -0
  124. misp_modules/modules/expansion/sigma_syntax_validator.py +49 -0
  125. misp_modules/modules/expansion/sigmf_expand.py +303 -0
  126. misp_modules/modules/expansion/socialscan.py +108 -0
  127. misp_modules/modules/expansion/sophoslabs_intelix.py +146 -0
  128. misp_modules/modules/expansion/sourcecache.py +57 -0
  129. misp_modules/modules/expansion/stairwell.py +156 -0
  130. misp_modules/modules/expansion/stix2_pattern_syntax_validator.py +56 -0
  131. misp_modules/modules/expansion/threatcrowd.py +170 -0
  132. misp_modules/modules/expansion/threatfox.py +75 -0
  133. misp_modules/modules/expansion/threatminer.py +152 -0
  134. misp_modules/modules/expansion/triage_submit.py +119 -0
  135. misp_modules/modules/expansion/trustar_enrich.py +235 -0
  136. misp_modules/modules/expansion/urlhaus.py +164 -0
  137. misp_modules/modules/expansion/urlscan.py +269 -0
  138. misp_modules/modules/expansion/variotdbs.py +133 -0
  139. misp_modules/modules/expansion/virustotal.py +311 -0
  140. misp_modules/modules/expansion/virustotal_public.py +272 -0
  141. misp_modules/modules/expansion/virustotal_upload.py +91 -0
  142. misp_modules/modules/expansion/vmray_submit.py +161 -0
  143. misp_modules/modules/expansion/vmware_nsx.py +628 -0
  144. misp_modules/modules/expansion/vulndb.py +292 -0
  145. misp_modules/modules/expansion/vulnerability_lookup.py +324 -0
  146. misp_modules/modules/expansion/vulners.py +81 -0
  147. misp_modules/modules/expansion/vysion.py +221 -0
  148. misp_modules/modules/expansion/whois.py +72 -0
  149. misp_modules/modules/expansion/whoisfreaks.py +232 -0
  150. misp_modules/modules/expansion/wiki.py +58 -0
  151. misp_modules/modules/expansion/xforceexchange.py +189 -0
  152. misp_modules/modules/expansion/xlsx_enrich.py +63 -0
  153. misp_modules/modules/expansion/yara_query.py +70 -0
  154. misp_modules/modules/expansion/yara_syntax_validator.py +49 -0
  155. misp_modules/modules/expansion/yeti.py +196 -0
  156. misp_modules/modules/export_mod/__init__.py +3 -0
  157. misp_modules/modules/export_mod/cef_export.py +93 -0
  158. misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +150 -0
  159. misp_modules/modules/export_mod/defender_endpoint_export.py +141 -0
  160. misp_modules/modules/export_mod/goamlexport.py +255 -0
  161. misp_modules/modules/export_mod/liteexport.py +97 -0
  162. misp_modules/modules/export_mod/mass_eql_export.py +106 -0
  163. misp_modules/modules/export_mod/nexthinkexport.py +131 -0
  164. misp_modules/modules/export_mod/osqueryexport.py +125 -0
  165. misp_modules/modules/export_mod/pdfexport.py +107 -0
  166. misp_modules/modules/export_mod/testexport.py +73 -0
  167. misp_modules/modules/export_mod/threatStream_misp_export.py +114 -0
  168. misp_modules/modules/export_mod/threat_connect_export.py +125 -0
  169. misp_modules/modules/export_mod/virustotal_collections.py +141 -0
  170. misp_modules/modules/export_mod/vt_graph.py +120 -0
  171. misp_modules/modules/export_mod/yara_export.py +290 -0
  172. misp_modules/modules/import_mod/__init__.py +22 -0
  173. misp_modules/modules/import_mod/cof2misp.py +264 -0
  174. misp_modules/modules/import_mod/csvimport.py +319 -0
  175. misp_modules/modules/import_mod/cuckooimport.py +749 -0
  176. misp_modules/modules/import_mod/email_import.py +294 -0
  177. misp_modules/modules/import_mod/goamlimport.py +188 -0
  178. misp_modules/modules/import_mod/import_blueprint.py +93 -0
  179. misp_modules/modules/import_mod/joe_import.py +74 -0
  180. misp_modules/modules/import_mod/lastline_import.py +160 -0
  181. misp_modules/modules/import_mod/mispjson.py +73 -0
  182. misp_modules/modules/import_mod/ocr.py +122 -0
  183. misp_modules/modules/import_mod/openiocimport.py +99 -0
  184. misp_modules/modules/import_mod/taxii21.py +383 -0
  185. misp_modules/modules/import_mod/testimport.py +73 -0
  186. misp_modules/modules/import_mod/threatanalyzer_import.py +552 -0
  187. misp_modules/modules/import_mod/url_import.py +97 -0
  188. misp_modules/modules/import_mod/vmray_import.py +96 -0
  189. misp_modules/modules/import_mod/vmray_summary_json_import.py +87 -0
  190. misp_modules-2.4.196.dist-info/LICENSE +661 -0
  191. misp_modules-2.4.196.dist-info/METADATA +281 -0
  192. misp_modules-2.4.196.dist-info/RECORD +194 -0
  193. misp_modules-2.4.196.dist-info/WHEEL +4 -0
  194. misp_modules-2.4.196.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,841 @@
1
+ """
2
+ Lastline Community API Client and Utilities.
3
+
4
+ :Copyright:
5
+ Copyright 2019 Lastline, Inc. All Rights Reserved.
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.
18
+
19
+ Copyright (c) 2010-2012 by Internet Systems Consortium, Inc. ("ISC")
20
+
21
+ Permission to use, copy, modify, and/or distribute this software for any
22
+ purpose with or without fee is hereby granted, provided that the above
23
+ copyright notice and this permission notice appear in all copies.
24
+
25
+ THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES
26
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
27
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR
28
+ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
29
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
30
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
31
+ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32
+ """
33
+ import abc
34
+ import logging
35
+ import io
36
+ import ipaddress
37
+ import pymisp
38
+ import re
39
+ import requests
40
+ from urllib import parse
41
+
42
+
43
+ DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi"
44
+
45
+ DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com"
46
+
47
+ LL_HOSTED_DOMAINS = frozenset([
48
+ "user.lastline.com",
49
+ "user.emea.lastline.com",
50
+ ])
51
+
52
+
53
+ def purge_none(d):
54
+ """Purge None entries from a dictionary."""
55
+ return {k: v for k, v in d.items() if v is not None}
56
+
57
+
58
+ def get_task_link(uuid, analysis_url=None, portal_url=None):
59
+ """
60
+ Get the task link given the task uuid and at least one API url.
61
+
62
+ :param str uuid: the task uuid
63
+ :param str|None analysis_url: the URL to the analysis API endpoint
64
+ :param str|None portal_url: the URL to the portal API endpoint
65
+ :rtype: str
66
+ :return: the task link
67
+ :raises ValueError: if not enough parameters have been provided
68
+ """
69
+ if not analysis_url and not portal_url:
70
+ raise ValueError("Neither analysis URL or portal URL have been specified")
71
+ if analysis_url:
72
+ portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user."))
73
+ portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid)
74
+ return parse.urljoin(portal_url, portal_url_path)
75
+
76
+
77
+ def get_portal_url_from_task_link(task_link):
78
+ """
79
+ Return the portal API url related to the provided task link.
80
+
81
+ :param str task_link: a link
82
+ :rtype: str
83
+ :return: the portal API url
84
+ """
85
+ parsed_uri = parse.urlparse(task_link)
86
+ return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri)
87
+
88
+
89
+ def get_uuid_from_task_link(task_link):
90
+ """
91
+ Return the uuid from a task link.
92
+
93
+ :param str task_link: a link
94
+ :rtype: str
95
+ :return: the uuid
96
+ :raises ValueError: if the link contains not task uuid
97
+ """
98
+ try:
99
+ return re.findall("[a-fA-F0-9]{32}", task_link)[0]
100
+ except IndexError:
101
+ raise ValueError("Link does not contain a valid task uuid")
102
+
103
+
104
+ def is_task_hosted(task_link):
105
+ """
106
+ Return whether the portal link is pointing to a hosted submission.
107
+
108
+ :param str task_link: a link
109
+ :rtype: boolean
110
+ :return: whether the link points to a hosted analysis
111
+ """
112
+ for domain in LL_HOSTED_DOMAINS:
113
+ if domain in task_link:
114
+ return True
115
+ return False
116
+
117
+
118
+ class InvalidArgument(Exception):
119
+ """Error raised invalid."""
120
+
121
+
122
+ class CommunicationError(Exception):
123
+ """Exception raised in case of timeouts or other network problem."""
124
+
125
+
126
+ class Error(Exception):
127
+ """Generic server error."""
128
+
129
+
130
+ class ApiError(Error):
131
+ """Server error with a message and an error code."""
132
+ def __init__(self, error_msg, error_code=None):
133
+ super(ApiError, self).__init__(error_msg, error_code)
134
+ self.error_msg = error_msg
135
+ self.error_code = error_code
136
+
137
+ def __str__(self):
138
+ if self.error_code is None:
139
+ error_code = ""
140
+ else:
141
+ error_code = " ({})".format(self.error_code)
142
+ return "{}{}".format(self.error_msg, error_code)
143
+
144
+
145
+ class LastlineAbstractClient(abc.ABC):
146
+ """"A very basic HTTP client providing basic functionality."""
147
+
148
+ __metaclass__ = abc.ABCMeta
149
+
150
+ SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login')
151
+ FORMATS = ["json", "xml"]
152
+
153
+ @classmethod
154
+ def sanitize_login_params(cls, api_key, api_token, username, password):
155
+ """
156
+ Return a dictionary with either API or USER credentials.
157
+
158
+ :param str|None api_key: the API key
159
+ :param str|None api_token: the API token
160
+ :param str|None username: the username
161
+ :param str|None password: the password
162
+ :rtype: dict[str, str]
163
+ :return: the dictionary
164
+ :raises InvalidArgument: if too many values are invalid
165
+ """
166
+ if api_key and api_token:
167
+ return {
168
+ "key": api_key,
169
+ "api_token": api_token,
170
+ }
171
+ elif username and password:
172
+ return {
173
+ "username": username,
174
+ "password": password,
175
+ }
176
+ else:
177
+ raise InvalidArgument("Arguments provided do not contain valid data")
178
+
179
+ @classmethod
180
+ def get_login_params_from_dict(cls, d):
181
+ """
182
+ Get the module configuration from a ConfigParser object.
183
+
184
+ :param dict[str, str] d: the dictionary
185
+ :rtype: dict[str, str]
186
+ :return: the parsed configuration
187
+ """
188
+ api_key = d.get("key")
189
+ api_token = d.get("api_token")
190
+ username = d.get("username")
191
+ password = d.get("password")
192
+ return cls.sanitize_login_params(api_key, api_token, username, password)
193
+
194
+ @classmethod
195
+ def get_login_params_from_conf(cls, conf, section_name):
196
+ """
197
+ Get the module configuration from a ConfigParser object.
198
+
199
+ :param ConfigParser conf: the conf object
200
+ :param str section_name: the section name
201
+ :rtype: dict[str, str]
202
+ :return: the parsed configuration
203
+ """
204
+ api_key = conf.get(section_name, "key", fallback=None)
205
+ api_token = conf.get(section_name, "api_token", fallback=None)
206
+ username = conf.get(section_name, "username", fallback=None)
207
+ password = conf.get(section_name, "password", fallback=None)
208
+ return cls.sanitize_login_params(api_key, api_token, username, password)
209
+
210
+ @classmethod
211
+ def load_from_conf(cls, conf, section_name):
212
+ """
213
+ Load client from a ConfigParser object.
214
+
215
+ :param ConfigParser conf: the conf object
216
+ :param str section_name: the section name
217
+ :rtype: T <- LastlineAbstractClient
218
+ :return: the loaded client
219
+ """
220
+ url = conf.get(section_name, "url")
221
+ return cls(url, cls.get_login_params_from_conf(conf, section_name))
222
+
223
+ def __init__(self, api_url, login_params, timeout=60, verify_ssl=True):
224
+ """
225
+ Instantiate a Lastline mini client.
226
+
227
+ :param str api_url: the URL of the API
228
+ :param dict[str, str]: the login parameters
229
+ :param int timeout: the timeout
230
+ :param boolean verify_ssl: whether to verify the SSL certificate
231
+ """
232
+ self._url = api_url
233
+ self._login_params = login_params
234
+ self._timeout = timeout
235
+ self._verify_ssl = verify_ssl
236
+ self._session = None
237
+ self._logger = logging.getLogger(__name__)
238
+
239
+ @abc.abstractmethod
240
+ def _login(self):
241
+ """Login using account-based or key-based methods."""
242
+
243
+ def _is_logged_in(self):
244
+ """Return whether we have an active session."""
245
+ return self._session is not None
246
+
247
+ @staticmethod
248
+ def _parse_response(response):
249
+ """
250
+ Parse the response.
251
+
252
+ :param requests.Response response: the response
253
+ :rtype: tuple(str|None, Error|ApiError)
254
+ :return: a tuple with mutually exclusive fields (either the response or the error)
255
+ """
256
+ try:
257
+ ret = response.json()
258
+ if "success" not in ret:
259
+ return None, Error("no success field in response")
260
+
261
+ if not ret["success"]:
262
+ error_msg = ret.get("error", "")
263
+ error_code = ret.get("error_code", None)
264
+ return None, ApiError(error_msg, error_code)
265
+
266
+ if "data" not in ret:
267
+ return None, Error("no data field in response")
268
+
269
+ return ret["data"], None
270
+ except ValueError as e:
271
+ return None, Error("Response not json {}".format(e))
272
+
273
+ def _handle_response(self, response, raw=False):
274
+ """
275
+ Check a response for issues and parse the return.
276
+
277
+ :param requests.Response response: the response
278
+ :param boolean raw: whether the raw body should be returned
279
+ :rtype: str
280
+ :return: if raw, return the response content; if not raw, the data field
281
+ :raises: CommunicationError, ApiError, Error
282
+ """
283
+ # Check for HTTP errors, and re-raise in case
284
+ try:
285
+ response.raise_for_status()
286
+ except requests.RequestException as e:
287
+ _, err = self._parse_response(response)
288
+ if isinstance(err, ApiError):
289
+ err_msg = "{}: {}".format(e, err.error_msg)
290
+ else:
291
+ err_msg = "{}".format(e)
292
+ raise CommunicationError(err_msg)
293
+
294
+ # Otherwise return the data (either parsed or not) but reraise if we have an API error
295
+ if raw:
296
+ return response.content
297
+ data, err = self._parse_response(response)
298
+ if err:
299
+ raise err
300
+ return data
301
+
302
+ def _build_url(self, sub_api, parts, requested_format="json"):
303
+ if sub_api not in self.SUB_APIS:
304
+ raise InvalidArgument(sub_api)
305
+ if requested_format not in self.FORMATS:
306
+ raise InvalidArgument(requested_format)
307
+ num_parts = 2 + len(parts)
308
+ pattern = "/".join(["%s"] * num_parts) + ".%s"
309
+ params = [self._url, sub_api] + parts + [requested_format]
310
+ return pattern % tuple(params)
311
+
312
+ def post(self, module, function, params=None, data=None, files=None, fmt="json"):
313
+ if isinstance(function, list):
314
+ functions = function
315
+ else:
316
+ functions = [function] if function else []
317
+ url = self._build_url(module, functions, requested_format=fmt)
318
+ return self.do_request(
319
+ url=url,
320
+ method="POST",
321
+ params=params,
322
+ data=data,
323
+ files=files,
324
+ fmt=fmt,
325
+ )
326
+
327
+ def get(self, module, function, params=None, fmt="json"):
328
+ if isinstance(function, list):
329
+ functions = function
330
+ else:
331
+ functions = [function] if function else []
332
+ url = self._build_url(module, functions, requested_format=fmt)
333
+ return self.do_request(
334
+ url=url,
335
+ method="GET",
336
+ params=params,
337
+ fmt=fmt,
338
+ )
339
+
340
+ def do_request(
341
+ self,
342
+ method,
343
+ url,
344
+ params=None,
345
+ data=None,
346
+ files=None,
347
+ fmt="json",
348
+ raw=False,
349
+ raw_response=False,
350
+ headers=None,
351
+ stream_response=False
352
+ ):
353
+ if raw_response:
354
+ raw = True
355
+
356
+ if fmt:
357
+ fmt = fmt.lower().strip()
358
+ if fmt not in self.FORMATS:
359
+ raise InvalidArgument("Only json, xml, html and pdf supported")
360
+ elif not raw:
361
+ raise InvalidArgument("Unformatted response requires raw=True")
362
+
363
+ if fmt != "json" and not raw:
364
+ raise InvalidArgument("Non-json format requires raw=True")
365
+
366
+ if method not in ["POST", "GET"]:
367
+ raise InvalidArgument("Only POST and GET supported")
368
+
369
+ if not self._is_logged_in():
370
+ self._login()
371
+
372
+ try:
373
+ try:
374
+ response = self._session.request(
375
+ method=method,
376
+ url=url,
377
+ data=data,
378
+ params=params,
379
+ files=files,
380
+ verify=self._verify_ssl,
381
+ timeout=self._timeout,
382
+ stream=stream_response,
383
+ headers=headers,
384
+ )
385
+ except requests.RequestException as e:
386
+ raise CommunicationError(e)
387
+
388
+ if raw_response:
389
+ return response
390
+ return self._handle_response(response, raw)
391
+
392
+ except Error as e:
393
+ raise e
394
+
395
+ except CommunicationError as e:
396
+ raise e
397
+
398
+
399
+ class AnalysisClient(LastlineAbstractClient):
400
+
401
+ def _login(self):
402
+ """
403
+ Creates auth session for malscape-service.
404
+
405
+ Credentials are 'key' and 'api_token'.
406
+ """
407
+ if self._session is None:
408
+ self._session = requests.session()
409
+ url = self._build_url("authentication", ["login"])
410
+ self.do_request("POST", url, params=purge_none(self._login_params))
411
+
412
+ def get_progress(self, uuid):
413
+ """
414
+ Get the completion progress of a given task.
415
+ :param str uuid: the unique identifier of the submitted task
416
+ :rtype: dict[str, int]
417
+ :return: a dictionary like the the following:
418
+ {
419
+ "completed": 1,
420
+ "progress": 100
421
+ }
422
+ """
423
+ url = self._build_url('analysis', ['get_progress'])
424
+ params = {'uuid': uuid}
425
+ return self.do_request("POST", url, params=params)
426
+
427
+ def get_result(self, uuid):
428
+ """
429
+ Get report results for a given task.
430
+
431
+ :param str uuid: the unique identifier of the submitted task
432
+ :rtype: dict[str, any]
433
+ :return: a dictionary like the the following:
434
+ {
435
+ "completed": 1,
436
+ "progress": 100
437
+ }
438
+ """
439
+ # better: use 'get_results()' but that would break
440
+ # backwards-compatibility
441
+ url = self._build_url('analysis', ['get'])
442
+ params = {'uuid': uuid}
443
+ return self.do_request("GET", url, params=params)
444
+
445
+ def submit_file(
446
+ self,
447
+ file_data,
448
+ file_name=None,
449
+ password=None,
450
+ analysis_env=None,
451
+ allow_network_traffic=True,
452
+ analysis_timeout=None,
453
+ bypass_cache=False,
454
+ ):
455
+ """
456
+ Upload a file to be analyzed.
457
+
458
+ :param bytes file_data: the data as a byte sequence
459
+ :param str|None file_name: if set, represents the name of the file to submit
460
+ :param str|None password: if set, use it to extract the sample
461
+ :param str|None analysis_env: if set, e.g windowsxp
462
+ :param boolean allow_network_traffic: if set to False, deny network connections
463
+ :param int|None analysis_timeout: if set limit the duration of the analysis
464
+ :param boolean bypass_cache: whether to re-process a file (requires special permissions)
465
+ :rtype: dict[str, any]
466
+ :return: a dictionary in the following form if the analysis is already available:
467
+ {
468
+ "submission": "2019-11-17 09:33:23",
469
+ "child_tasks": [...],
470
+ "reports": [...],
471
+ "submission_timestamp": "2019-11-18 16:11:04",
472
+ "task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
473
+ "score": 0,
474
+ "analysis_subject": {
475
+ "url": "https://www.google.com"
476
+ },
477
+ "last_submission_timestamp": "2019-11-18 16:11:04"
478
+ }
479
+
480
+ OR the following if the analysis is still pending:
481
+
482
+ {
483
+ "submission_timestamp": "2019-11-18 13:59:25",
484
+ "task_uuid": "f3c0ae115d51001017ff8da768fa6049",
485
+ }
486
+ """
487
+ file_stream = io.BytesIO(file_data)
488
+ api_url = self._build_url("analysis", ["submit", "file"])
489
+ params = purge_none({
490
+ "bypass_cache": bypass_cache and 1 or None,
491
+ "analysis_timeout": analysis_timeout,
492
+ "analysis_env": analysis_env,
493
+ "allow_network_traffic": allow_network_traffic and 1 or None,
494
+ "filename": file_name,
495
+ "password": password,
496
+ "full_report_score": -1,
497
+ })
498
+
499
+ files = purge_none({
500
+ # If an explicit filename was provided, we can pass it down to
501
+ # python-requests to use it in the multipart/form-data. This avoids
502
+ # having python-requests trying to guess the filename based on stream
503
+ # attributes.
504
+ #
505
+ # The problem with this is that, if the filename is not ASCII, then
506
+ # this triggers a bug in flask/werkzeug which means the file is
507
+ # thrown away. Thus, we just force an ASCII name
508
+ "file": ('dummy-ascii-name-for-file-param', file_stream),
509
+ })
510
+
511
+ return self.do_request("POST", api_url, params=params, files=files)
512
+
513
+ def submit_url(
514
+ self,
515
+ url,
516
+ referer=None,
517
+ user_agent=None,
518
+ bypass_cache=False,
519
+ ):
520
+ """
521
+ Upload an URL to be analyzed.
522
+
523
+ :param str url: the url to analyze
524
+ :param str|None referer: the referer
525
+ :param str|None user_agent: the user agent
526
+ :param boolean bypass_cache: bypass_cache
527
+ :rtype: dict[str, any]
528
+ :return: a dictionary like the following if the analysis is already available:
529
+ {
530
+ "submission": "2019-11-17 09:33:23",
531
+ "child_tasks": [...],
532
+ "reports": [...],
533
+ "submission_timestamp": "2019-11-18 16:11:04",
534
+ "task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
535
+ "score": 0,
536
+ "analysis_subject": {
537
+ "url": "https://www.google.com"
538
+ },
539
+ "last_submission_timestamp": "2019-11-18 16:11:04"
540
+ }
541
+
542
+ OR the following if the analysis is still pending:
543
+
544
+ {
545
+ "submission_timestamp": "2019-11-18 13:59:25",
546
+ "task_uuid": "f3c0ae115d51001017ff8da768fa6049",
547
+ }
548
+ """
549
+ api_url = self._build_url("analysis", ["submit", "url"])
550
+ params = purge_none({
551
+ "url": url,
552
+ "referer": referer,
553
+ "bypass_cache": bypass_cache and 1 or None,
554
+ "user_agent": user_agent or None,
555
+ })
556
+ return self.do_request("POST", api_url, params=params)
557
+
558
+
559
+ class PortalClient(LastlineAbstractClient):
560
+
561
+ def _login(self):
562
+ """
563
+ Login using account-based or key-based methods.
564
+
565
+ Credentials are 'username' and 'password'
566
+ """
567
+ if self._session is None:
568
+ self._session = requests.session()
569
+ self.post("login", function=None, data=self._login_params)
570
+
571
+ def get_progress(self, uuid, analysis_instance=None):
572
+ """
573
+ Get the completion progress of a given task.
574
+
575
+ :param str uuid: the unique identifier of the submitted task
576
+ :param str analysis_instance: if set, defines the analysis instance to query
577
+ :rtype: dict[str, int]
578
+ :return: a dictionary like the the following:
579
+ {
580
+ "completed": 1,
581
+ "progress": 100
582
+ }
583
+ """
584
+ params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance})
585
+ return self.get("analysis", "get_progress", params=params)
586
+
587
+ def get_result(self, uuid, analysis_instance=None):
588
+ """
589
+ Get report results for a given task.
590
+
591
+ :param str uuid: the unique identifier of the submitted task
592
+ :param str analysis_instance: if set, defines the analysis instance to query
593
+ :rtype: dict[str, any]
594
+ :return: a dictionary like the the following:
595
+ {
596
+ "completed": 1,
597
+ "progress": 100
598
+ }
599
+ """
600
+ params = purge_none(
601
+ {
602
+ "uuid": uuid,
603
+ "analysis_instance": analysis_instance,
604
+ "report_format": "json",
605
+ }
606
+ )
607
+ return self.get("analysis", "get_result", params=params)
608
+
609
+ def submit_url(
610
+ self,
611
+ url,
612
+ referer=None,
613
+ user_agent=None,
614
+ bypass_cache=False,
615
+ ):
616
+ """
617
+ Upload an URL to be analyzed.
618
+
619
+ :param str url: the url to analyze
620
+ :param str|None referer: the referer
621
+ :param str|None user_agent: the user agent
622
+ :param boolean bypass_cache: bypass_cache
623
+ :rtype: dict[str, any]
624
+ :return: a dictionary like the following if the analysis is already available:
625
+ {
626
+ "submission": "2019-11-17 09:33:23",
627
+ "child_tasks": [...],
628
+ "reports": [...],
629
+ "submission_timestamp": "2019-11-18 16:11:04",
630
+ "task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
631
+ "score": 0,
632
+ "analysis_subject": {
633
+ "url": "https://www.google.com"
634
+ },
635
+ "last_submission_timestamp": "2019-11-18 16:11:04"
636
+ }
637
+
638
+ OR the following if the analysis is still pending:
639
+
640
+ {
641
+ "submission_timestamp": "2019-11-18 13:59:25",
642
+ "task_uuid": "f3c0ae115d51001017ff8da768fa6049",
643
+ }
644
+ """
645
+ params = purge_none(
646
+ {
647
+ "url": url,
648
+ "bypass_cache": bypass_cache,
649
+ "referer": referer,
650
+ "user_agent": user_agent
651
+ }
652
+ )
653
+ return self.post("analysis", "submit_url", params=params)
654
+
655
+ def submit_file(
656
+ self,
657
+ file_data,
658
+ file_name=None,
659
+ password=None,
660
+ analysis_env=None,
661
+ allow_network_traffic=True,
662
+ analysis_timeout=None,
663
+ bypass_cache=False,
664
+ ):
665
+ """
666
+ Upload a file to be analyzed.
667
+
668
+ :param bytes file_data: the data as a byte sequence
669
+ :param str|None file_name: if set, represents the name of the file to submit
670
+ :param str|None password: if set, use it to extract the sample
671
+ :param str|None analysis_env: if set, e.g windowsxp
672
+ :param boolean allow_network_traffic: if set to False, deny network connections
673
+ :param int|None analysis_timeout: if set limit the duration of the analysis
674
+ :param boolean bypass_cache: whether to re-process a file (requires special permissions)
675
+ :rtype: dict[str, any]
676
+ :return: a dictionary in the following form if the analysis is already available:
677
+ {
678
+ "submission": "2019-11-17 09:33:23",
679
+ "child_tasks": [...],
680
+ "reports": [...],
681
+ "submission_timestamp": "2019-11-18 16:11:04",
682
+ "task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
683
+ "score": 0,
684
+ "analysis_subject": {
685
+ "url": "https://www.google.com"
686
+ },
687
+ "last_submission_timestamp": "2019-11-18 16:11:04"
688
+ }
689
+
690
+ OR the following if the analysis is still pending:
691
+
692
+ {
693
+ "submission_timestamp": "2019-11-18 13:59:25",
694
+ "task_uuid": "f3c0ae115d51001017ff8da768fa6049",
695
+ }
696
+ """
697
+ params = purge_none(
698
+ {
699
+ "filename": file_name,
700
+ "password": password,
701
+ "analysis_env": analysis_env,
702
+ "allow_network_traffic": allow_network_traffic,
703
+ "analysis_timeout": analysis_timeout,
704
+ "bypass_cache": bypass_cache,
705
+ }
706
+ )
707
+ files = {"file": (file_name, file_data, "application/octet-stream")}
708
+ return self.post("analysis", "submit_file", params=params, files=files)
709
+
710
+
711
+ class LastlineResultBaseParser(object):
712
+ """
713
+ This is a parser to extract *basic* information from a Lastline result dictionary.
714
+
715
+ Note: this is a *version 0*: the information we extract is merely related to the behaviors and
716
+ the HTTP connections. Further iterations will include host activity such as files, mutexes,
717
+ registry keys, strings, etc.
718
+ """
719
+
720
+ def __init__(self):
721
+ """Constructor."""
722
+ self.misp_event = None
723
+
724
+ @staticmethod
725
+ def _get_mitre_techniques(result):
726
+ return [
727
+ "misp-galaxy:mitre-attack-pattern=\"{} - {}\"".format(w[0], w[1])
728
+ for w in sorted(set([
729
+ (y["id"], y["name"])
730
+ for x in result.get("malicious_activity", [])
731
+ for y in result.get("activity_to_mitre_techniques", {}).get(x, [])
732
+ ]))
733
+ ]
734
+
735
+ def parse(self, analysis_link, result):
736
+ """
737
+ Parse the analysis result into a MISP event.
738
+
739
+ :param str analysis_link: the analysis link
740
+ :param dict[str, any] result: the JSON returned by the analysis client.
741
+ :rtype: MISPEvent
742
+ :return: some results that can be consumed by MIPS.
743
+ """
744
+ self.misp_event = pymisp.MISPEvent()
745
+
746
+ # Add analysis subject info
747
+ if "url" in result["analysis_subject"]:
748
+ o = pymisp.MISPObject("url")
749
+ o.add_attribute("url", result["analysis_subject"]["url"])
750
+ else:
751
+ o = pymisp.MISPObject("file")
752
+ o.add_attribute("md5", type="md5", value=result["analysis_subject"]["md5"])
753
+ o.add_attribute("sha1", type="sha1", value=result["analysis_subject"]["sha1"])
754
+ o.add_attribute("sha256", type="sha256", value=result["analysis_subject"]["sha256"])
755
+ o.add_attribute(
756
+ "mimetype",
757
+ type="mime-type",
758
+ value=result["analysis_subject"]["mime_type"]
759
+ )
760
+ self.misp_event.add_object(o)
761
+
762
+ # Add HTTP requests from url analyses
763
+ network_dict = result.get("report", {}).get("analysis", {}).get("network", {})
764
+ for request in network_dict.get("requests", []):
765
+ parsed_uri = parse.urlparse(request["url"])
766
+ o = pymisp.MISPObject(name='http-request')
767
+ o.add_attribute('host', parsed_uri.netloc)
768
+ o.add_attribute('method', "GET")
769
+ o.add_attribute('uri', request["url"])
770
+ o.add_attribute("ip", request["ip"])
771
+ self.misp_event.add_object(o)
772
+
773
+ # Add network behaviors from files
774
+ for subject in result.get("report", {}).get("analysis_subjects", []):
775
+
776
+ # Add DNS requests
777
+ for dns_query in subject.get("dns_queries", []):
778
+ hostname = dns_query.get("hostname")
779
+ # Skip if it is an IP address
780
+ try:
781
+ if hostname == "wpad":
782
+ continue
783
+ _ = ipaddress.ip_address(hostname)
784
+ continue
785
+ except ValueError:
786
+ pass
787
+
788
+ o = pymisp.MISPObject(name='dns-record')
789
+ o.add_attribute('queried-domain', hostname)
790
+ self.misp_event.add_object(o)
791
+
792
+ # Add HTTP conversations (as network connection and as http request)
793
+ for http_conversation in subject.get("http_conversations", []):
794
+ o = pymisp.MISPObject(name="network-connection")
795
+ o.add_attribute("ip-src", http_conversation["src_ip"])
796
+ o.add_attribute("ip-dst", http_conversation["dst_ip"])
797
+ o.add_attribute("src-port", http_conversation["src_port"])
798
+ o.add_attribute("dst-port", http_conversation["dst_port"])
799
+ o.add_attribute("hostname-dst", http_conversation["dst_host"])
800
+ o.add_attribute("layer3-protocol", "IP")
801
+ o.add_attribute("layer4-protocol", "TCP")
802
+ o.add_attribute("layer7-protocol", "HTTP")
803
+ self.misp_event.add_object(o)
804
+
805
+ method, path, http_version = http_conversation["url"].split(" ")
806
+ if http_conversation["dst_port"] == 80:
807
+ uri = "http://{}{}".format(http_conversation["dst_host"], path)
808
+ else:
809
+ uri = "http://{}:{}{}".format(
810
+ http_conversation["dst_host"],
811
+ http_conversation["dst_port"],
812
+ path
813
+ )
814
+ o = pymisp.MISPObject(name='http-request')
815
+ o.add_attribute('host', http_conversation["dst_host"])
816
+ o.add_attribute('method', method)
817
+ o.add_attribute('uri', uri)
818
+ o.add_attribute('ip', http_conversation["dst_ip"])
819
+ self.misp_event.add_object(o)
820
+
821
+ # Add sandbox info like score and sandbox type
822
+ o = pymisp.MISPObject(name="sandbox-report")
823
+ sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise"
824
+ o.add_attribute("score", result["score"])
825
+ o.add_attribute("sandbox-type", sandbox_type)
826
+ o.add_attribute("{}-sandbox".format(sandbox_type), "lastline")
827
+ o.add_attribute("permalink", analysis_link)
828
+ self.misp_event.add_object(o)
829
+
830
+ # Add behaviors
831
+ o = pymisp.MISPObject(name="sb-signature")
832
+ o.add_attribute("software", "Lastline")
833
+ for activity in result.get("malicious_activity", []):
834
+ a = pymisp.MISPAttribute()
835
+ a.from_dict(type="text", value=activity)
836
+ o.add_attribute("signature", **a)
837
+ self.misp_event.add_object(o)
838
+
839
+ # Add mitre techniques
840
+ for technique in self._get_mitre_techniques(result):
841
+ self.misp_event.add_tag(technique)