dshield-parser 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dshield_parser/__init__.py +1 -0
- dshield_parser/__main__.py +20 -0
- dshield_parser/cowrie_processor/__init__.py +1 -0
- dshield_parser/cowrie_processor/__main__.py +0 -0
- dshield_parser/cowrie_processor/reports.py +79 -0
- dshield_parser/firewall_processor/__init__.py +1 -0
- dshield_parser/firewall_processor/__main__.py +0 -0
- dshield_parser/firewall_processor/reports.py +79 -0
- dshield_parser/test/__init__.py +0 -0
- dshield_parser/test/test_json.py +356 -0
- dshield_parser/utils/__init__.py +1 -0
- dshield_parser/utils/__main__.py +0 -0
- dshield_parser/utils/enrichment.py +458 -0
- dshield_parser/utils/file_io.py +108 -0
- dshield_parser/utils/graphs.py +121 -0
- dshield_parser/utils/json.py +508 -0
- dshield_parser/utils/ml.py +69 -0
- dshield_parser/utils/sql.py +204 -0
- dshield_parser/utils/ssh.py +0 -0
- dshield_parser/web_processor/__init__.py +1 -0
- dshield_parser/web_processor/__main__.py +0 -0
- dshield_parser/web_processor/reports.py +76 -0
- dshield_parser-0.0.0.dist-info/METADATA +3 -0
- dshield_parser-0.0.0.dist-info/RECORD +26 -0
- dshield_parser-0.0.0.dist-info/WHEEL +5 -0
- dshield_parser-0.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
import logging, datetime
|
|
2
|
+
import sys
|
|
3
|
+
import re
|
|
4
|
+
import json
|
|
5
|
+
import requests #needs to be installed
|
|
6
|
+
import shodan #needs to be installed
|
|
7
|
+
import dshield_parser.utils as utils
|
|
8
|
+
import dshield_parser.utils.json as dshield_json
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import sys
|
|
11
|
+
import os
|
|
12
|
+
import time
|
|
13
|
+
import ipaddress
|
|
14
|
+
import requests
|
|
15
|
+
import xml.etree.ElementTree as ET
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from functools import lru_cache
|
|
18
|
+
import datetime
|
|
19
|
+
import socket
|
|
20
|
+
|
|
21
|
+
#Enter a valid IP Address (e.g. 1.1.1.1), ASN (e.g. AS1234), or TAG (e.g. NORD_VPN)
|
|
22
|
+
@lru_cache
|
|
23
|
+
def get_spur_data(log_source, input, session, api):
|
|
24
|
+
try:
|
|
25
|
+
ipaddress.ip_address(input)
|
|
26
|
+
logging.debug(f"IP Address: {input}")
|
|
27
|
+
except Exception as e:
|
|
28
|
+
logging.debug(f"Not an IP Address: {input}")
|
|
29
|
+
|
|
30
|
+
session.headers = {'Token': api}
|
|
31
|
+
url = "https://api.spur.us/v2/context/" + input
|
|
32
|
+
while True:
|
|
33
|
+
try:
|
|
34
|
+
response = session.get(url)
|
|
35
|
+
except Exception as e:
|
|
36
|
+
logging.error(f"Exception hit for SPUR query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
37
|
+
time.sleep(15)
|
|
38
|
+
continue
|
|
39
|
+
break
|
|
40
|
+
json_data = json.loads(response.text)
|
|
41
|
+
json_data["DShield Log Source"] = log_source
|
|
42
|
+
json_data["DShield Log Identifier"] = input
|
|
43
|
+
file = open("spur_data.json", 'a',encoding="utf-8")
|
|
44
|
+
file.write(json.dumps(json_data) + "\n")
|
|
45
|
+
file.close()
|
|
46
|
+
return (json_data)
|
|
47
|
+
|
|
48
|
+
#URL, IP Address, Domain or file hash
|
|
49
|
+
@lru_cache
|
|
50
|
+
def get_vt_data(input, input_type, api):
|
|
51
|
+
vt_data = {}
|
|
52
|
+
headers = {'X-Apikey': api}
|
|
53
|
+
if input_type == "url":
|
|
54
|
+
url = "https://www.virustotal.com/api/v3/urls/" + input #url
|
|
55
|
+
elif input_type == "domain":
|
|
56
|
+
url = "https://www.virustotal.com/api/v3/domains/" + input #domain
|
|
57
|
+
elif input_type == "hash":
|
|
58
|
+
url = "https://www.virustotal.com/api/v3/files/" + input #hash
|
|
59
|
+
elif input_type == "ip":
|
|
60
|
+
url = "https://www.virustotal.com/api/v3/ip_addresses/" + input #ip address
|
|
61
|
+
else:
|
|
62
|
+
logging.error("Invalid input type given for VirusTotal to process.")
|
|
63
|
+
|
|
64
|
+
while True:
|
|
65
|
+
try:
|
|
66
|
+
response = requests.get(url, headers=headers)
|
|
67
|
+
except Exception as e:
|
|
68
|
+
logging.error(f"Exception hit for VirusTotal query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
69
|
+
time.sleep(15)
|
|
70
|
+
continue
|
|
71
|
+
break
|
|
72
|
+
|
|
73
|
+
json_response = json.loads(response.text)
|
|
74
|
+
logging.debug(response.text)
|
|
75
|
+
if "error" in json_response:
|
|
76
|
+
for key, value in json_response["error"].items():
|
|
77
|
+
logging.error(f"VT Error for '{hash}': {value}")
|
|
78
|
+
elif "data" in json_response:
|
|
79
|
+
if "attributes" in json_response["data"]:
|
|
80
|
+
if "last_analysis_stats" in json_response["data"]["attributes"]:
|
|
81
|
+
for key, value in json_response["data"]["attributes"]["last_analysis_stats"].items():
|
|
82
|
+
vt_data[key] = value
|
|
83
|
+
if "last_analysis_date" in json_response["data"]["attributes"]:
|
|
84
|
+
vt_data["last_analysis_date"] = json_response["data"]["attributes"]["last_analysis_date"]
|
|
85
|
+
if "trid" in json_response["data"]["attributes"]:
|
|
86
|
+
vt_data["filetype"] = json_response["data"]["attributes"]["trid"][0]["file_type"]
|
|
87
|
+
if "type_tag" in json_response["data"]["attributes"]:
|
|
88
|
+
vt_data["typetag"] = json_response["data"]["attributes"]["type_tag"]
|
|
89
|
+
if "type_description" in json_response["data"]["attributes"]:
|
|
90
|
+
vt_data["description"] = json_response["data"]["attributes"]["type_description"]
|
|
91
|
+
if "meaningful_name" in json_response["data"]["attributes"]:
|
|
92
|
+
vt_data["filename"] = json_response["data"]["attributes"]["meaningful_name"]
|
|
93
|
+
if "popular_threat_classification" in json_response["data"]["attributes"]:
|
|
94
|
+
vt_data["classification"] = json_response["data"]["attributes"]["popular_threat_classification"]["suggested_threat_label"]
|
|
95
|
+
|
|
96
|
+
filehandle = open("vt_data.json", "a", encoding="utf-8")
|
|
97
|
+
filehandle.write(json.dumps(vt_data) +"\n")
|
|
98
|
+
filehandle.close()
|
|
99
|
+
return(vt_data)
|
|
100
|
+
|
|
101
|
+
@lru_cache
|
|
102
|
+
def get_uh_data(input):
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
@lru_cache
|
|
106
|
+
def get_whois_data(input):
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
@lru_cache
|
|
110
|
+
def get_isc_data(input):
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
@lru_cache
|
|
114
|
+
def get_shodan_host(input, key):
|
|
115
|
+
api_shodan = shodan.Shodan(key)
|
|
116
|
+
try:
|
|
117
|
+
info = api_shodan.host(input)
|
|
118
|
+
return info
|
|
119
|
+
except Exception as e:
|
|
120
|
+
logging.error(f"Issue looking up {input}: {e}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@lru_cache
|
|
124
|
+
def get_shodan_domain(input, key):
|
|
125
|
+
api_shodan = shodan.Shodan(key)
|
|
126
|
+
try:
|
|
127
|
+
info = api_shodan.dns.domain_info(input)
|
|
128
|
+
return info
|
|
129
|
+
except Exception as e:
|
|
130
|
+
logging.error(f"Issue looking up {input}: {e}")
|
|
131
|
+
|
|
132
|
+
#https://proxycheck.io/
|
|
133
|
+
@lru_cache
|
|
134
|
+
def get_proxycheck_data(input):
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
#https://ipinfo.io/
|
|
138
|
+
@lru_cache
|
|
139
|
+
def get_ipinfo_data(input):
|
|
140
|
+
pass
|
|
141
|
+
|
|
142
|
+
#https://www.abuseipdb.com/
|
|
143
|
+
@lru_cache
|
|
144
|
+
def get_abuseipdb_data(input):
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
#https://cybergordon.com/
|
|
148
|
+
@lru_cache
|
|
149
|
+
def get_cybergordon_data(input):
|
|
150
|
+
pass
|
|
151
|
+
|
|
152
|
+
@lru_cache
|
|
153
|
+
def get_urlscan_data(input):
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
@lru_cache
|
|
157
|
+
def get_input(input):
|
|
158
|
+
if "http" in input:
|
|
159
|
+
logging.debug(f"URL: {input}")
|
|
160
|
+
elif "." in input:
|
|
161
|
+
try:
|
|
162
|
+
ipaddress.ip_address(input)
|
|
163
|
+
logging.debug(f"IP Address: {input}")
|
|
164
|
+
except Exception as e:
|
|
165
|
+
logging.debug(f"Not an IP Address: {input}")
|
|
166
|
+
|
|
167
|
+
@lru_cache
|
|
168
|
+
def extract_ip_address(input):
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
def isc_cloudapis(email):
|
|
172
|
+
url = "https://isc.sans.edu/api/cloudips"
|
|
173
|
+
headers = {
|
|
174
|
+
'User-Agent': f'Request from {email}',
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
while True:
|
|
178
|
+
try:
|
|
179
|
+
response = requests.get(url, headers=headers)
|
|
180
|
+
except Exception as e:
|
|
181
|
+
logging.error(f"Exception hit for Could IP SANS ISC query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
182
|
+
time.sleep(15)
|
|
183
|
+
continue
|
|
184
|
+
break
|
|
185
|
+
|
|
186
|
+
xml = response.text
|
|
187
|
+
root = ET.fromstring(xml)
|
|
188
|
+
ips = []
|
|
189
|
+
netmasks = []
|
|
190
|
+
providers = []
|
|
191
|
+
comments = []
|
|
192
|
+
for idx, cidr in enumerate(root.findall("cidr")):
|
|
193
|
+
for idx2, ip in enumerate(cidr.findall("ip")):
|
|
194
|
+
#print("IP: ", ip.text)
|
|
195
|
+
ips.append(ip.text)
|
|
196
|
+
for idx2, netmask in enumerate(cidr.findall("netmask")):
|
|
197
|
+
#print("Netmask: ", netmask.text)
|
|
198
|
+
netmasks.append(netmask.text)
|
|
199
|
+
for idx2, provider in enumerate(cidr.findall("provider")):
|
|
200
|
+
#print("Provider: ", provider.text)
|
|
201
|
+
providers.append(provider.text)
|
|
202
|
+
for idx2, comment in enumerate(cidr.findall("comment")):
|
|
203
|
+
#print("Comment: ", comment.text)
|
|
204
|
+
comments.append(comment.text)
|
|
205
|
+
d = {'IP Address': ips, 'Netmask': netmasks, 'Provider': providers, 'Comment': comments}
|
|
206
|
+
return pd.DataFrame(d)
|
|
207
|
+
|
|
208
|
+
def isc_cloudcidrs(email):
|
|
209
|
+
url = "https://isc.sans.edu/api/cloudcidrs"
|
|
210
|
+
headers = {
|
|
211
|
+
'User-Agent': f'Request from {email}',
|
|
212
|
+
}
|
|
213
|
+
while True:
|
|
214
|
+
try:
|
|
215
|
+
response = requests.get(url, headers=headers)
|
|
216
|
+
except Exception as e:
|
|
217
|
+
logging.error(f"Exception hit for Cloud CIDR SANS ISC query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
218
|
+
time.sleep(15)
|
|
219
|
+
continue
|
|
220
|
+
break
|
|
221
|
+
|
|
222
|
+
xml = response.text
|
|
223
|
+
root = ET.fromstring(xml)
|
|
224
|
+
ips = []
|
|
225
|
+
providers = []
|
|
226
|
+
comments = []
|
|
227
|
+
for idx, cidr in enumerate(root.findall("cidr")):
|
|
228
|
+
for idx2, ip in enumerate(cidr.findall("prefix")):
|
|
229
|
+
#print("IP: ", ip.text)
|
|
230
|
+
ips.append(ip.text)
|
|
231
|
+
for idx2, provider in enumerate(cidr.findall("provider")):
|
|
232
|
+
#print("Provider: ", provider.text)
|
|
233
|
+
providers.append(provider.text)
|
|
234
|
+
for idx2, comment in enumerate(cidr.findall("comment")):
|
|
235
|
+
#print("Comment: ", comment.text)
|
|
236
|
+
comments.append(comment.text)
|
|
237
|
+
d = {'CIDR': ips, 'Provider': providers, 'Comment': comments}
|
|
238
|
+
return pd.DataFrame(d)
|
|
239
|
+
|
|
240
|
+
def isc_intelfeed(email):
|
|
241
|
+
url = "https://isc.sans.edu/api/intelfeed"
|
|
242
|
+
headers = {
|
|
243
|
+
f'User-Agent': 'Request from {email}',
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
while True:
|
|
247
|
+
try:
|
|
248
|
+
response = requests.get(url, headers=headers)
|
|
249
|
+
except Exception as e:
|
|
250
|
+
logging.error(f"Exception hit for SANS ISC Intel Feed query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
251
|
+
time.sleep(15)
|
|
252
|
+
continue
|
|
253
|
+
break
|
|
254
|
+
|
|
255
|
+
xml = response.text
|
|
256
|
+
root = ET.fromstring(xml)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
#return pd.DataFrame(d)
|
|
260
|
+
|
|
261
|
+
@lru_cache
|
|
262
|
+
def isc_ipinfo(ip, email):
|
|
263
|
+
|
|
264
|
+
url = f"https://isc.sans.edu/api/ip/{ip}"
|
|
265
|
+
headers = {
|
|
266
|
+
'User-Agent': f'Request from {email}',
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
while True:
|
|
270
|
+
try:
|
|
271
|
+
response = requests.get(url, headers=headers)
|
|
272
|
+
except Exception as e:
|
|
273
|
+
logging.error(f"Exception hit for SANS ISC IP info query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
274
|
+
time.sleep(15)
|
|
275
|
+
continue
|
|
276
|
+
break
|
|
277
|
+
|
|
278
|
+
while response.status_code != 200:
|
|
279
|
+
delay = 5
|
|
280
|
+
if response.status_code == 429:
|
|
281
|
+
logging.error(f"Request limit reached: {response.text}")
|
|
282
|
+
try:
|
|
283
|
+
delay_received = int(re.findall(r'.*Try again after (.*) seconds', response.text)[0])
|
|
284
|
+
delay = int(delay_received)
|
|
285
|
+
logging.error(f"Delaying for an additional {delay} seconds")
|
|
286
|
+
except:
|
|
287
|
+
logging.error(f"Some issue occured with the delay we recevied: {delay_received}")
|
|
288
|
+
time.sleep(delay)
|
|
289
|
+
response = requests.get(url, headers=headers)
|
|
290
|
+
if response.status_code == 200:
|
|
291
|
+
xml = response.text
|
|
292
|
+
logging.debug(f"XML Data: {xml}")
|
|
293
|
+
root = ET.fromstring(xml)
|
|
294
|
+
ipdata = {}
|
|
295
|
+
ipdata[ip] = {}
|
|
296
|
+
ipdata[ip]["number"] = root.findall("number")[0].text
|
|
297
|
+
ipdata[ip]["count"] = root.findall("count")[0].text
|
|
298
|
+
ipdata[ip]["attacks"] = root.findall("attacks")[0].text
|
|
299
|
+
ipdata[ip]["maxdate"] = root.findall("maxdate")[0].text
|
|
300
|
+
ipdata[ip]["mindate"] = root.findall("mindate")[0].text
|
|
301
|
+
ipdata[ip]["updated"] = root.findall("updated")[0].text
|
|
302
|
+
ipdata[ip]["comment"] = root.findall("comment")[0].text
|
|
303
|
+
ipdata[ip]["maxrisk"] = root.findall("maxrisk")[0].text
|
|
304
|
+
ipdata[ip]["asabusecontact"] = root.findall("asabusecontact")[0].text
|
|
305
|
+
ipdata[ip]["as"] = root.findall("as")[0].text
|
|
306
|
+
ipdata[ip]["asname"] = root.findall("asname")[0].text
|
|
307
|
+
ipdata[ip]["ascountry"] = root.findall("ascountry")[0].text
|
|
308
|
+
ipdata[ip]["assize"] = root.findall("assize")[0].text
|
|
309
|
+
ipdata[ip]["network"] = root.findall("network")[0].text
|
|
310
|
+
for idx2, webloginfo in enumerate(root.findall("weblogs")):
|
|
311
|
+
ipdata[ip]["weblog_count"] = webloginfo.findall("count")[0].text
|
|
312
|
+
ipdata[ip]["weblog_avgauthors"] = webloginfo.findall("avgauthors")[0].text
|
|
313
|
+
ipdata[ip]["weblog_avgurls"] = webloginfo.findall("avgurls")[0].text
|
|
314
|
+
ipdata[ip]["weblog_avguseragents"] = webloginfo.findall("avguser_agents")[0].text
|
|
315
|
+
ipdata[ip]["weblog_firstseen"] = webloginfo.findall("firstseen")[0].text
|
|
316
|
+
ipdata[ip]["lastseen"] = webloginfo.findall("lastseen")[0].text
|
|
317
|
+
return ipdata
|
|
318
|
+
|
|
319
|
+
#<threatcategory>
|
|
320
|
+
#<research>
|
|
321
|
+
#<ipv4>4.16.74.81</ipv4>
|
|
322
|
+
#<added>2020-07-18</added>
|
|
323
|
+
#<lastseen>2024-07-24</lastseen>
|
|
324
|
+
#<type>scorecard</type>
|
|
325
|
+
#</research>
|
|
326
|
+
def isc_research_threats(email, start_date=None, end_date=None):
|
|
327
|
+
#url = f"https://isc.sans.edu/api/ip/{ip}"
|
|
328
|
+
date_range = ""
|
|
329
|
+
if start_date is not None and end_date is not None:
|
|
330
|
+
date_range = f"{start_date}/{end_date}"
|
|
331
|
+
url = f"https://isc.sans.edu/api/threatcategory/research/{date_range}"
|
|
332
|
+
headers = {
|
|
333
|
+
'User-Agent': f'Request from {email}',
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
while True:
|
|
337
|
+
try:
|
|
338
|
+
response = requests.get(url, headers=headers)
|
|
339
|
+
except Exception as e:
|
|
340
|
+
logging.error(f"Exception hit for SANS ISC Research Threats query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
341
|
+
time.sleep(15)
|
|
342
|
+
continue
|
|
343
|
+
break
|
|
344
|
+
|
|
345
|
+
xml = response.text
|
|
346
|
+
logging.debug(f"XML Data: {xml}")
|
|
347
|
+
root = ET.fromstring(xml)
|
|
348
|
+
threat_intel = []
|
|
349
|
+
for idx, research_info in enumerate(root.findall("research")):
|
|
350
|
+
threat_intel.append(
|
|
351
|
+
{
|
|
352
|
+
"ipv4": research_info.findall("ipv4")[0].text,
|
|
353
|
+
"added": research_info.findall("added")[0].text,
|
|
354
|
+
"lastseen": research_info.findall("lastseen")[0].text,
|
|
355
|
+
"type": research_info.findall("type")[0].text
|
|
356
|
+
}
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
return pd.DataFrame(threat_intel)
|
|
360
|
+
|
|
361
|
+
def binary_edge_minions(email):
|
|
362
|
+
# bad URL as of 7/6/2025
|
|
363
|
+
# url = f"https://api.binaryedge.io/krang/v1/minions"
|
|
364
|
+
url = f"https://api.binaryedge.io/v1/minions"
|
|
365
|
+
|
|
366
|
+
headers = {
|
|
367
|
+
'User-Agent': f'Request from {email}',
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
while True:
|
|
371
|
+
try:
|
|
372
|
+
#this gave error 404 on 7/6/2025
|
|
373
|
+
response = requests.get(url, headers=headers)
|
|
374
|
+
except Exception as e:
|
|
375
|
+
logging.error(f"Exception hit for Binary Edge query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
376
|
+
time.sleep(15)
|
|
377
|
+
continue
|
|
378
|
+
break
|
|
379
|
+
|
|
380
|
+
scanners = json.loads(response.text)['scanners']
|
|
381
|
+
|
|
382
|
+
#return a list of binary edge minions (scanning agents)
|
|
383
|
+
return scanners
|
|
384
|
+
|
|
385
|
+
def censys_networks(email):
|
|
386
|
+
url = f"https://support.censys.io/hc/en-us/article_attachments/25644686434196"
|
|
387
|
+
|
|
388
|
+
headers = {
|
|
389
|
+
'User-Agent': f'Request from {email}',
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
while True:
|
|
393
|
+
try:
|
|
394
|
+
response = requests.get(url, headers=headers)
|
|
395
|
+
except Exception as e:
|
|
396
|
+
logging.error(f"Exception hit for Censys query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
397
|
+
time.sleep(15)
|
|
398
|
+
continue
|
|
399
|
+
break
|
|
400
|
+
|
|
401
|
+
scanners = response.text.split("\n")
|
|
402
|
+
|
|
403
|
+
#return a list of binary edge minions (scanning agents)
|
|
404
|
+
return scanners
|
|
405
|
+
|
|
406
|
+
@lru_cache
|
|
407
|
+
def get_reverse_dns(ipaddress):
|
|
408
|
+
response = ""
|
|
409
|
+
try:
|
|
410
|
+
response = socket.gethostbyaddr(f"{ipaddress}")
|
|
411
|
+
except Exception as e:
|
|
412
|
+
if "host not found" in e:
|
|
413
|
+
logging.info(f"Lookup failed for {ipaddress}: {e}")
|
|
414
|
+
else:
|
|
415
|
+
logging.error(f"Lookup failed for {ipaddress}: {e}")
|
|
416
|
+
if len(response) == 0:
|
|
417
|
+
return response
|
|
418
|
+
else:
|
|
419
|
+
return response[0]
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def isc_webhoneypot_useragents(user_agent, day, email):
|
|
423
|
+
url = f"https://isc.sans.edu/api/webhoneypotreportsbyua/{user_agent}/{day}?json"
|
|
424
|
+
headers = {
|
|
425
|
+
'User-Agent': f'Request from {email}',
|
|
426
|
+
}
|
|
427
|
+
while True:
|
|
428
|
+
try:
|
|
429
|
+
response = requests.get(url, headers=headers)
|
|
430
|
+
except Exception as e:
|
|
431
|
+
logging.error(f"Exception hit for SANS ISC web honeypot user agent query: '{e}'. Pausing for 15 seconds before trying again.")
|
|
432
|
+
time.sleep(15)
|
|
433
|
+
continue
|
|
434
|
+
break
|
|
435
|
+
|
|
436
|
+
return json.loads(response.text)
|
|
437
|
+
|
|
438
|
+
def extract_ips(data):
|
|
439
|
+
ips = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', data)
|
|
440
|
+
return_ips = []
|
|
441
|
+
for each_ip in ips:
|
|
442
|
+
try:
|
|
443
|
+
ip = ipaddress.ip_address(each_ip)
|
|
444
|
+
return_ips.append(each_ip)
|
|
445
|
+
except:
|
|
446
|
+
logging.error(f"Error trying to create IP address object from {each_ip}")
|
|
447
|
+
return return_ips
|
|
448
|
+
|
|
449
|
+
def extract_subnets(data):
|
|
450
|
+
networks = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})', data)
|
|
451
|
+
return_networks = []
|
|
452
|
+
for each_network in networks:
|
|
453
|
+
try:
|
|
454
|
+
network = ipaddress.ip_network(each_network)
|
|
455
|
+
return_networks.append(each_network)
|
|
456
|
+
except:
|
|
457
|
+
logging.error(f"Error trying to create network object from {each_network}")
|
|
458
|
+
return return_networks
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import json
|
|
3
|
+
import dshield_parser.utils as utils
|
|
4
|
+
import dshield_parser.utils.json as dshield_json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import sys
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
def read_file_string(filename):
|
|
10
|
+
filehandle = open(filename, encoding="utf-8")
|
|
11
|
+
string = filehandle.read()
|
|
12
|
+
return string
|
|
13
|
+
|
|
14
|
+
def read_file_json(filename):
|
|
15
|
+
try:
|
|
16
|
+
filehandle = open(filename, encoding="utf-8")
|
|
17
|
+
json_data = json.load(filehandle)
|
|
18
|
+
except:
|
|
19
|
+
json_data = read_file_json_stream(filename)
|
|
20
|
+
return json_data
|
|
21
|
+
|
|
22
|
+
def read_file_json_stream(filename):
|
|
23
|
+
data = []
|
|
24
|
+
with open(filename) as file:
|
|
25
|
+
for line in file:
|
|
26
|
+
if line[:1] =="{":
|
|
27
|
+
data.append(json.loads(line))
|
|
28
|
+
return(data)
|
|
29
|
+
|
|
30
|
+
def read_file_json_generator(filename):
|
|
31
|
+
file = open(filename, "r", encoding="utf-8")
|
|
32
|
+
for line in file:
|
|
33
|
+
if line[:1] =="{":
|
|
34
|
+
try:
|
|
35
|
+
yield json.loads(line) #.replace('\x00','') # added .replace('\x00','') on 8/17/24 to replace null bytes for pandas merge issues
|
|
36
|
+
except Exception as e:
|
|
37
|
+
logging.error(f"Exception hit for file '{filename}': {e}")
|
|
38
|
+
logging.error(f"Text of line with error (limted to first 40 characters): '{line[0:40]}'")
|
|
39
|
+
file.close()
|
|
40
|
+
|
|
41
|
+
def get_honeypot_list(path):
|
|
42
|
+
honeypots = []
|
|
43
|
+
dir_contents = os.listdir(path)
|
|
44
|
+
for each_item in dir_contents:
|
|
45
|
+
if os.path.isdir(path + "\\" + each_item):
|
|
46
|
+
honeypots.append(each_item)
|
|
47
|
+
return honeypots
|
|
48
|
+
|
|
49
|
+
def get_file_lists(path):
|
|
50
|
+
cowrie_files = []
|
|
51
|
+
web_honeypot_files = []
|
|
52
|
+
firewall_files = []
|
|
53
|
+
if os.path.exists(f"{path}\\logs\\dshield_firewall_.log"):
|
|
54
|
+
firewall_files.append(f"{path}\\logs\\dshield_firewall_.log")
|
|
55
|
+
try:
|
|
56
|
+
folder_list = Path(path)
|
|
57
|
+
#file_list = sorted(Path(path).iterdir(), key=os.path.getmtime)
|
|
58
|
+
except:
|
|
59
|
+
print(f"{path}: File Listing Error Occurred")
|
|
60
|
+
for each_item in folder_list.glob('**/*'):
|
|
61
|
+
if "cowrie.json" in str(each_item):
|
|
62
|
+
cowrie_files.append(each_item)
|
|
63
|
+
elif "webhoneypot-" in str(each_item):
|
|
64
|
+
web_honeypot_files.append(each_item)
|
|
65
|
+
elif "webhoneypot_" in str(each_item):
|
|
66
|
+
web_honeypot_files.append(each_item)
|
|
67
|
+
cowrie_files = sorted(cowrie_files, key=os.path.getmtime)
|
|
68
|
+
web_honeypot_files = sorted(web_honeypot_files, key=os.path.getmtime)
|
|
69
|
+
firewall_files = sorted(firewall_files, key=os.path.getmtime)
|
|
70
|
+
return cowrie_files, web_honeypot_files, firewall_files
|
|
71
|
+
|
|
72
|
+
def get_overlapping_file_lists(path):
|
|
73
|
+
cowrie_files, web_honeypot_files, firewall_files = get_file_lists(path)
|
|
74
|
+
|
|
75
|
+
def get_honeypot_directories(path):
|
|
76
|
+
honeypots = []
|
|
77
|
+
honeypot_dirs = []
|
|
78
|
+
try:
|
|
79
|
+
file_list = sorted(Path(path).iterdir(), key=os.path.getmtime)
|
|
80
|
+
except:
|
|
81
|
+
print("File Listing Error Occurred")
|
|
82
|
+
for each_item in file_list:
|
|
83
|
+
if os.path.isdir(each_item):
|
|
84
|
+
honeypots.append(os.path.basename(each_item))
|
|
85
|
+
honeypot_dirs.append(each_item)
|
|
86
|
+
return honeypots, honeypot_dirs
|
|
87
|
+
|
|
88
|
+
def save_dict_to_file(dict, filename):
|
|
89
|
+
logging.info(f"Saving data from dict to file '{filename}'")
|
|
90
|
+
filehandle = open(filename, "a", encoding="utf-8")
|
|
91
|
+
for key, value in dict.items():
|
|
92
|
+
filehandle.write(f"{value}\t{key}\n")
|
|
93
|
+
filehandle.close()
|
|
94
|
+
|
|
95
|
+
def save_list_to_file(list, filename):
|
|
96
|
+
logging.info(f"Saving data from list to file '{filename}'")
|
|
97
|
+
filehandle = open(filename, "a", encoding="utf-8")
|
|
98
|
+
for each_item in dict:
|
|
99
|
+
filehandle.write(f"{each_item}\n")
|
|
100
|
+
filehandle.close()
|
|
101
|
+
|
|
102
|
+
# will be used when executing the script
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
filepath = sys.argv[1]
|
|
105
|
+
if os.path.exists(filepath):
|
|
106
|
+
dshield_json.print_json_pretty(sys.argv[1])
|
|
107
|
+
else:
|
|
108
|
+
print("Argument supplied is not a file that exists.")
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import matplotlib.pyplot as plt
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
#def plot_date_counts(dict):
|
|
7
|
+
# lists = sorted(dict.items())
|
|
8
|
+
# x, y = zip(*lists)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
#plt.locator_params(axis='y', nticks=4)
|
|
12
|
+
#plt.locator_params(axis='x', nticks=8)
|
|
13
|
+
# plt.ticklabel_format(style='plain')
|
|
14
|
+
# plt.plot(x, y)
|
|
15
|
+
# plt.xticks(get_ticks(x, 8))
|
|
16
|
+
# plt.ylabel("Number of Access Attempts")
|
|
17
|
+
# plt.xlabel("Date")
|
|
18
|
+
# plt.title("Honeypot Traffic Over Time")
|
|
19
|
+
# plt.show()
|
|
20
|
+
|
|
21
|
+
def get_ticks(values, numticks):
|
|
22
|
+
ticks = []
|
|
23
|
+
length = len(values)
|
|
24
|
+
ticks.append(values[0])
|
|
25
|
+
separation = int((length) / (numticks -1))
|
|
26
|
+
counter = separation
|
|
27
|
+
while counter < (length -1):
|
|
28
|
+
ticks.append(values[counter])
|
|
29
|
+
counter += separation
|
|
30
|
+
if len(ticks) < numticks:
|
|
31
|
+
ticks.append(values[length-1])
|
|
32
|
+
return ticks
|
|
33
|
+
|
|
34
|
+
def plot_date_counts(data, main_title, y_label):
|
|
35
|
+
if len(data) > 1:
|
|
36
|
+
num_rows = (len(data) // 2) + (len(data) % 2)
|
|
37
|
+
num_columns = (len(data) // num_rows) + (len(data) % 2)
|
|
38
|
+
figure, subplots = plt.subplots(num_rows, num_columns, figsize=(20,12), constrained_layout=True)
|
|
39
|
+
else:
|
|
40
|
+
figure, subplots = plt.subplots(figsize=(10,6), constrained_layout=True)
|
|
41
|
+
|
|
42
|
+
plots = subplots.flatten()
|
|
43
|
+
|
|
44
|
+
#overlapping_figure, overlapping_plot = plt.subplots(figsize=(20,12), constrained_layout=True)
|
|
45
|
+
#overlapping_plot.set_ylabel(y_label)
|
|
46
|
+
#overlapping_plot.set_xlabel("Date")
|
|
47
|
+
#overlapping_plot.set_title(main_title)
|
|
48
|
+
|
|
49
|
+
plot_index = 0
|
|
50
|
+
|
|
51
|
+
figure.suptitle(main_title)
|
|
52
|
+
|
|
53
|
+
for honeypotname, honeypot_data in data.items():
|
|
54
|
+
if len(honeypot_data) == 0:
|
|
55
|
+
logging.info("Blank data received for {honeypotname}")
|
|
56
|
+
logging.info("Will ignore plotting graph data for {honeypotname}")
|
|
57
|
+
else:
|
|
58
|
+
lists = sorted(honeypot_data.items())
|
|
59
|
+
x, y = zip(*lists)
|
|
60
|
+
|
|
61
|
+
if len(data) == 1:
|
|
62
|
+
subplots.xaxis.set_major_locator(plt.MaxNLocator(8))
|
|
63
|
+
subplots.ticklabel_format(style='plain')
|
|
64
|
+
subplots.plot(x, y)
|
|
65
|
+
subplots.set_ylabel(y_label)
|
|
66
|
+
subplots.set_xlabel("Date")
|
|
67
|
+
subplots.set_title(honeypotname)
|
|
68
|
+
else:
|
|
69
|
+
plots[plot_index].xaxis.set_major_locator(plt.MaxNLocator(8))
|
|
70
|
+
#ax.ticklabel_format(style='plain')
|
|
71
|
+
plots[plot_index].plot(x, y)
|
|
72
|
+
#ax.plot([1, 2, 3], label='Inline label')
|
|
73
|
+
#line.set_label('Label via method')
|
|
74
|
+
#ax.legend()
|
|
75
|
+
#overlapping_plot.plot(x, y, label=honeypotname)
|
|
76
|
+
plots[plot_index].set_ylabel(y_label)
|
|
77
|
+
plots[plot_index].set_xlabel("Date")
|
|
78
|
+
plots[plot_index].set_title(honeypotname)
|
|
79
|
+
plot_index += 1
|
|
80
|
+
|
|
81
|
+
#plt.show()
|
|
82
|
+
plt.tight_layout()
|
|
83
|
+
plt.savefig(f'{main_title.replace(" ","_")}_multi.png')
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
#overlapping_plot.legend()
|
|
87
|
+
#overlapping_plot.plt.savefig(f'{main_title.replace(" ","_")}_overlap.png')
|
|
88
|
+
|
|
89
|
+
def plot_date_counts_overlapping(data, main_title, y_label):
|
|
90
|
+
overlapping_figure, overlapping_plot = plt.subplots(figsize=(20,12), constrained_layout=True)
|
|
91
|
+
overlapping_plot.set_ylabel(y_label)
|
|
92
|
+
overlapping_plot.set_xlabel("Date")
|
|
93
|
+
overlapping_plot.set_title(main_title)
|
|
94
|
+
for honeypotname, honeypot_data in data.items():
|
|
95
|
+
if len(honeypot_data) == 0:
|
|
96
|
+
logging.info("Blank data received for {honeypotname}")
|
|
97
|
+
logging.info("Will ignore plotting graph data for {honeypotname}")
|
|
98
|
+
else:
|
|
99
|
+
lists = sorted(honeypot_data.items())
|
|
100
|
+
x, y = zip(*lists)
|
|
101
|
+
overlapping_plot.plot(x, y, label=honeypotname)
|
|
102
|
+
overlapping_plot.legend()
|
|
103
|
+
overlapping_plot.xaxis.set_major_locator(plt.MaxNLocator(8))
|
|
104
|
+
plt.tight_layout()
|
|
105
|
+
plt.savefig(f'{main_title.replace(" ","_")}_overlapping.png')
|
|
106
|
+
|
|
107
|
+
def plot_time_counts(dict, plottype=None, ylabel=None, xlabel=None, title=None, type=None):
|
|
108
|
+
lists = sorted(dict.items())
|
|
109
|
+
x, y = zip(*lists)
|
|
110
|
+
start = x[0]
|
|
111
|
+
end = x[-1]
|
|
112
|
+
#daterange = pd.timedelta_range(start, end)
|
|
113
|
+
|
|
114
|
+
plt.ticklabel_format(style='plain')
|
|
115
|
+
plt.hist(x, y)
|
|
116
|
+
#plt.xticks(daterange)
|
|
117
|
+
plt.ylabel("Number of Access Attempts")
|
|
118
|
+
plt.xlabel("Date")
|
|
119
|
+
plt.title("Honeypot Traffic Over Time")
|
|
120
|
+
plt.show()
|
|
121
|
+
|