bbot 2.4.0.6045rc0__py3-none-any.whl → 2.4.0.6067rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of bbot might be problematic. Click here for more details.
- bbot/__init__.py +1 -1
- bbot/cli.py +1 -1
- bbot/core/event/base.py +60 -20
- bbot/core/event/helpers.py +222 -36
- bbot/core/helpers/helper.py +1 -1
- bbot/core/helpers/regexes.py +1 -1
- bbot/core/helpers/web/web.py +1 -1
- bbot/modules/deadly/nuclei.py +3 -1
- bbot/modules/internal/excavate.py +1 -1
- bbot/modules/output/asset_inventory.py +1 -1
- bbot/modules/portscan.py +2 -13
- bbot/scanner/manager.py +30 -16
- bbot/scanner/preset/preset.py +1 -5
- bbot/scanner/scanner.py +7 -2
- bbot/scanner/target.py +90 -148
- bbot/test/test_step_1/test_cli.py +1 -1
- bbot/test/test_step_1/test_event_seeds.py +160 -0
- bbot/test/test_step_1/test_presets.py +3 -3
- bbot/test/test_step_1/test_python_api.py +1 -1
- bbot/test/test_step_1/test_regexes.py +20 -13
- bbot/test/test_step_1/test_scan.py +1 -1
- bbot/test/test_step_1/test_target.py +12 -14
- bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py +2 -1
- {bbot-2.4.0.6045rc0.dist-info → bbot-2.4.0.6067rc0.dist-info}/METADATA +1 -1
- {bbot-2.4.0.6045rc0.dist-info → bbot-2.4.0.6067rc0.dist-info}/RECORD +28 -27
- {bbot-2.4.0.6045rc0.dist-info → bbot-2.4.0.6067rc0.dist-info}/LICENSE +0 -0
- {bbot-2.4.0.6045rc0.dist-info → bbot-2.4.0.6067rc0.dist-info}/WHEEL +0 -0
- {bbot-2.4.0.6045rc0.dist-info → bbot-2.4.0.6067rc0.dist-info}/entry_points.txt +0 -0
bbot/scanner/scanner.py
CHANGED
|
@@ -365,7 +365,7 @@ class Scanner:
|
|
|
365
365
|
|
|
366
366
|
# distribute seed events
|
|
367
367
|
self.init_events_task = asyncio.create_task(
|
|
368
|
-
self.ingress_module.init_events(self.target.seeds.
|
|
368
|
+
self.ingress_module.init_events(self.target.seeds.event_seeds),
|
|
369
369
|
name=f"{self.name}.ingress_module.init_events()",
|
|
370
370
|
)
|
|
371
371
|
|
|
@@ -1022,6 +1022,7 @@ class Scanner:
|
|
|
1022
1022
|
root_event._id = self.id
|
|
1023
1023
|
root_event.scope_distance = 0
|
|
1024
1024
|
root_event.parent = root_event
|
|
1025
|
+
root_event._dummy = False
|
|
1025
1026
|
root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET")
|
|
1026
1027
|
return root_event
|
|
1027
1028
|
|
|
@@ -1301,7 +1302,11 @@ class Scanner:
|
|
|
1301
1302
|
try:
|
|
1302
1303
|
yield
|
|
1303
1304
|
except BaseException as e:
|
|
1304
|
-
|
|
1305
|
+
try:
|
|
1306
|
+
self._handle_exception(e, context=context, unhandled_is_critical=unhandled_is_critical)
|
|
1307
|
+
except Exception as e2:
|
|
1308
|
+
self.log.critical(f"Error in exception handler: {e2} {traceback.format_exc()}")
|
|
1309
|
+
raise
|
|
1305
1310
|
|
|
1306
1311
|
def _handle_exception(self, e, context="scan", finally_callback=None, unhandled_is_critical=False):
|
|
1307
1312
|
if callable(context):
|
bbot/scanner/target.py
CHANGED
|
@@ -5,26 +5,19 @@ from radixtarget import RadixTarget
|
|
|
5
5
|
from radixtarget.helpers import host_size_key
|
|
6
6
|
|
|
7
7
|
from bbot.errors import *
|
|
8
|
-
from bbot.core.event import
|
|
9
|
-
from bbot.core.helpers
|
|
10
|
-
|
|
8
|
+
from bbot.core.event import is_event
|
|
9
|
+
from bbot.core.event.helpers import EventSeed, BaseEventSeed
|
|
10
|
+
from bbot.core.helpers.misc import is_dns_name, is_ip, is_ip_type
|
|
11
11
|
|
|
12
12
|
log = logging.getLogger("bbot.core.target")
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
def special_target_type(regex_pattern):
|
|
16
|
-
def decorator(func):
|
|
17
|
-
func._regex = re.compile(regex_pattern, re.IGNORECASE)
|
|
18
|
-
return func
|
|
19
|
-
|
|
20
|
-
return decorator
|
|
21
|
-
|
|
22
|
-
|
|
23
15
|
class BaseTarget(RadixTarget):
|
|
24
16
|
"""
|
|
25
17
|
A collection of BBOT events that represent a scan target.
|
|
26
18
|
|
|
27
|
-
|
|
19
|
+
The purpose of this class is to hold a potentially huge target list in a space-efficient way,
|
|
20
|
+
while allowing lightning fast scope lookups.
|
|
28
21
|
|
|
29
22
|
This class is inherited by all three components of the BBOT target:
|
|
30
23
|
- Whitelist
|
|
@@ -32,89 +25,75 @@ class BaseTarget(RadixTarget):
|
|
|
32
25
|
- Seeds
|
|
33
26
|
"""
|
|
34
27
|
|
|
35
|
-
|
|
36
|
-
# regex-callback pairs for handling special target types
|
|
37
|
-
# these aren't defined explicitly; instead they are decorated with @special_target_type
|
|
38
|
-
# the function must return a list of events
|
|
39
|
-
}
|
|
40
|
-
tags = []
|
|
41
|
-
|
|
42
|
-
def __init__(self, *targets, scan=None, **kwargs):
|
|
43
|
-
self.scan = scan
|
|
44
|
-
self.events = set()
|
|
45
|
-
self.inputs = set()
|
|
46
|
-
# Register decorated methods
|
|
47
|
-
for method in dir(self):
|
|
48
|
-
if callable(getattr(self, method, None)):
|
|
49
|
-
func = getattr(self, method)
|
|
50
|
-
if hasattr(func, "_regex"):
|
|
51
|
-
self.special_target_types[func._regex] = func
|
|
28
|
+
accept_target_types = ["TARGET"]
|
|
52
29
|
|
|
30
|
+
def __init__(self, *targets, **kwargs):
|
|
31
|
+
self.event_seeds = set()
|
|
53
32
|
super().__init__(*targets, **kwargs)
|
|
54
33
|
|
|
34
|
+
@property
|
|
35
|
+
def inputs(self):
|
|
36
|
+
return set(e.input for e in self.event_seeds)
|
|
37
|
+
|
|
55
38
|
def get(self, event, **kwargs):
|
|
56
39
|
"""
|
|
57
|
-
|
|
40
|
+
Here we override RadixTarget's get() method, which normally only accepts hosts, to also accept events for convenience.
|
|
58
41
|
"""
|
|
59
|
-
|
|
42
|
+
host = None
|
|
43
|
+
raise_error = kwargs.get("raise_error", False)
|
|
44
|
+
# if it's already an event or event seed, use its host
|
|
45
|
+
if is_event(event) or isinstance(event, BaseEventSeed):
|
|
60
46
|
host = event.host
|
|
61
47
|
# save resources by checking if the event is an IP or DNS name
|
|
62
48
|
elif is_ip(event, include_network=True) or is_dns_name(event):
|
|
63
49
|
host = event
|
|
50
|
+
# if it's a string, autodetect its type and parse out its host
|
|
64
51
|
elif isinstance(event, str):
|
|
65
|
-
|
|
66
|
-
host =
|
|
52
|
+
event_seed = self._make_event_seed(event, raise_error=raise_error)
|
|
53
|
+
host = event_seed.host
|
|
54
|
+
if not host:
|
|
55
|
+
return
|
|
67
56
|
else:
|
|
68
|
-
raise ValueError(f"Invalid
|
|
57
|
+
raise ValueError(f"Invalid target type for {self.__class__.__name__}: {type(event)}")
|
|
69
58
|
if not host:
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
59
|
+
msg = f"Host not found: '{event}'"
|
|
60
|
+
if raise_error:
|
|
61
|
+
raise KeyError(msg)
|
|
62
|
+
else:
|
|
63
|
+
log.warning(msg)
|
|
64
|
+
return
|
|
73
65
|
results = super().get(host, **kwargs)
|
|
74
66
|
return results
|
|
75
67
|
|
|
76
|
-
def
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def add(self, targets):
|
|
68
|
+
def _make_event_seed(self, target, raise_error=False):
|
|
69
|
+
try:
|
|
70
|
+
return EventSeed(target)
|
|
71
|
+
except ValidationError:
|
|
72
|
+
msg = f"Invalid target: '{target}'"
|
|
73
|
+
if raise_error:
|
|
74
|
+
raise KeyError(msg)
|
|
75
|
+
else:
|
|
76
|
+
log.warning(msg)
|
|
77
|
+
|
|
78
|
+
def add(self, targets, data=None):
|
|
87
79
|
if not isinstance(targets, (list, set, tuple)):
|
|
88
80
|
targets = [targets]
|
|
89
|
-
|
|
81
|
+
event_seeds = set()
|
|
90
82
|
for target in targets:
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
event = self.make_event(target)
|
|
97
|
-
if event:
|
|
98
|
-
self.inputs.add(str(target))
|
|
99
|
-
_events = [event]
|
|
100
|
-
for event in _events:
|
|
101
|
-
events.add(event)
|
|
83
|
+
event_seed = EventSeed(target)
|
|
84
|
+
if not event_seed._target_type in self.accept_target_types:
|
|
85
|
+
log.warning(f"Invalid target type for {self.__class__.__name__}: {event_seed.type}")
|
|
86
|
+
continue
|
|
87
|
+
event_seeds.add(event_seed)
|
|
102
88
|
|
|
103
89
|
# sort by host size to ensure consistency
|
|
104
|
-
|
|
105
|
-
for
|
|
106
|
-
self.
|
|
107
|
-
self._add(
|
|
108
|
-
|
|
109
|
-
def check_special_target_types(self, target):
|
|
110
|
-
for regex, callback in self.special_target_types.items():
|
|
111
|
-
match = regex.match(target)
|
|
112
|
-
if match:
|
|
113
|
-
return True, callback(match)
|
|
114
|
-
return False, []
|
|
90
|
+
event_seeds = sorted(event_seeds, key=lambda e: ((0, 0) if not e.host else host_size_key(e.host)))
|
|
91
|
+
for event_seed in event_seeds:
|
|
92
|
+
self.event_seeds.add(event_seed)
|
|
93
|
+
self._add(event_seed.host, data=(event_seed if data is None else data))
|
|
115
94
|
|
|
116
95
|
def __iter__(self):
|
|
117
|
-
yield from self.
|
|
96
|
+
yield from self.event_seeds
|
|
118
97
|
|
|
119
98
|
|
|
120
99
|
class ScanSeeds(BaseTarget):
|
|
@@ -124,36 +103,6 @@ class ScanSeeds(BaseTarget):
|
|
|
124
103
|
These are the targets specified by the user, e.g. via `-t` on the CLI.
|
|
125
104
|
"""
|
|
126
105
|
|
|
127
|
-
tags = ["target"]
|
|
128
|
-
|
|
129
|
-
@special_target_type(r"^(?:ORG|ORG_STUB):(.*)")
|
|
130
|
-
def handle_org_stub(self, match):
|
|
131
|
-
org_stub_event = self.make_event(match.group(1), event_type="ORG_STUB")
|
|
132
|
-
if org_stub_event:
|
|
133
|
-
return [org_stub_event]
|
|
134
|
-
return []
|
|
135
|
-
|
|
136
|
-
@special_target_type(r"^(?:USER|USERNAME):(.*)")
|
|
137
|
-
def handle_username(self, match):
|
|
138
|
-
username_event = self.make_event(match.group(1), event_type="USERNAME")
|
|
139
|
-
if username_event:
|
|
140
|
-
return [username_event]
|
|
141
|
-
return []
|
|
142
|
-
|
|
143
|
-
@special_target_type(r"^(?:FILESYSTEM|FILE|FOLDER|DIR|PATH):(.*)")
|
|
144
|
-
def handle_filesystem(self, match):
|
|
145
|
-
filesystem_event = self.make_event({"path": match.group(1)}, event_type="FILESYSTEM")
|
|
146
|
-
if filesystem_event:
|
|
147
|
-
return [filesystem_event]
|
|
148
|
-
return []
|
|
149
|
-
|
|
150
|
-
@special_target_type(r"^(?:MOBILE_APP|APK|IPA|APP):(.*)")
|
|
151
|
-
def handle_mobile_app(self, match):
|
|
152
|
-
mobile_app_event = self.make_event({"url": match.group(1)}, event_type="MOBILE_APP")
|
|
153
|
-
if mobile_app_event:
|
|
154
|
-
return [mobile_app_event]
|
|
155
|
-
return []
|
|
156
|
-
|
|
157
106
|
def get(self, event, single=True, **kwargs):
|
|
158
107
|
results = super().get(event, **kwargs)
|
|
159
108
|
if results and single:
|
|
@@ -165,6 +114,9 @@ class ScanSeeds(BaseTarget):
|
|
|
165
114
|
Overrides the base method to enable having multiple events for the same host.
|
|
166
115
|
|
|
167
116
|
The "data" attribute of the node is now a set of events.
|
|
117
|
+
|
|
118
|
+
This is useful for seeds, because it lets us have both evilcorp.com:80 and https://evilcorp.com
|
|
119
|
+
as separate events even though they have the same host.
|
|
168
120
|
"""
|
|
169
121
|
if host:
|
|
170
122
|
try:
|
|
@@ -176,7 +128,7 @@ class ScanSeeds(BaseTarget):
|
|
|
176
128
|
|
|
177
129
|
def _hash_value(self):
|
|
178
130
|
# seeds get hashed by event data
|
|
179
|
-
return sorted(str(e.data).encode() for e in self.
|
|
131
|
+
return sorted(str(e.data).encode() for e in self.event_seeds)
|
|
180
132
|
|
|
181
133
|
|
|
182
134
|
class ACLTarget(BaseTarget):
|
|
@@ -199,39 +151,46 @@ class ScanBlacklist(ACLTarget):
|
|
|
199
151
|
A collection of BBOT events that represent a scan's blacklist.
|
|
200
152
|
"""
|
|
201
153
|
|
|
154
|
+
accept_target_types = ["TARGET", "BLACKLIST"]
|
|
155
|
+
|
|
202
156
|
def __init__(self, *args, **kwargs):
|
|
203
157
|
self.blacklist_regexes = set()
|
|
204
158
|
super().__init__(*args, **kwargs)
|
|
205
159
|
|
|
206
|
-
|
|
207
|
-
def handle_regex(self, match):
|
|
208
|
-
pattern = match.group(1)
|
|
209
|
-
blacklist_regex = re.compile(pattern, re.IGNORECASE)
|
|
210
|
-
self.blacklist_regexes.add(blacklist_regex)
|
|
211
|
-
return []
|
|
212
|
-
|
|
213
|
-
def get(self, event, **kwargs):
|
|
160
|
+
def get(self, host, **kwargs):
|
|
214
161
|
"""
|
|
215
|
-
|
|
162
|
+
Blacklists only accept IPs or strings. This is cleaner since we need to search for regex patterns.
|
|
216
163
|
"""
|
|
217
|
-
|
|
164
|
+
if not (is_ip_type(host) or isinstance(host, str)):
|
|
165
|
+
raise ValueError(f"Invalid target type for {self.__class__.__name__}: {type(host)}")
|
|
166
|
+
raise_error = kwargs.get("raise_error", False)
|
|
218
167
|
# first, check event's host against blacklist
|
|
219
168
|
try:
|
|
220
|
-
|
|
169
|
+
event_seed = self._make_event_seed(host, raise_error=raise_error)
|
|
170
|
+
host = event_seed.host
|
|
171
|
+
to_match = event_seed.data
|
|
172
|
+
except ValidationError:
|
|
173
|
+
to_match = str(host)
|
|
174
|
+
try:
|
|
175
|
+
event_result = super().get(host, raise_error=True)
|
|
221
176
|
except KeyError:
|
|
222
177
|
event_result = None
|
|
223
178
|
if event_result is not None:
|
|
224
179
|
return event_result
|
|
225
180
|
# next, check event's host against regexes
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
if kwargs.get("raise_error", False):
|
|
232
|
-
raise KeyError(f"Host not found: '{event.data}'")
|
|
181
|
+
for regex in self.blacklist_regexes:
|
|
182
|
+
if regex.search(to_match):
|
|
183
|
+
return host
|
|
184
|
+
if raise_error:
|
|
185
|
+
raise KeyError(f"Host not found: '{host}'")
|
|
233
186
|
return None
|
|
234
187
|
|
|
188
|
+
def _add(self, host, data):
|
|
189
|
+
if getattr(data, "type", "") == "BLACKLIST_REGEX":
|
|
190
|
+
self.blacklist_regexes.add(re.compile(data.data))
|
|
191
|
+
if host is not None:
|
|
192
|
+
super()._add(host, data)
|
|
193
|
+
|
|
235
194
|
def _hash_value(self):
|
|
236
195
|
# regexes are included in blacklist hash
|
|
237
196
|
regex_patterns = [str(r.pattern).encode() for r in self.blacklist_regexes]
|
|
@@ -255,23 +214,22 @@ class BBOTTarget:
|
|
|
255
214
|
Provides high-level functions like in_scope(), which includes both whitelist and blacklist checks.
|
|
256
215
|
"""
|
|
257
216
|
|
|
258
|
-
def __init__(self, *seeds, whitelist=None, blacklist=None, strict_scope=False
|
|
259
|
-
self.scan = scan
|
|
217
|
+
def __init__(self, *seeds, whitelist=None, blacklist=None, strict_scope=False):
|
|
260
218
|
self.strict_scope = strict_scope
|
|
261
|
-
self.seeds = ScanSeeds(*seeds, strict_dns_scope=strict_scope
|
|
219
|
+
self.seeds = ScanSeeds(*seeds, strict_dns_scope=strict_scope)
|
|
262
220
|
if whitelist is None:
|
|
263
221
|
whitelist = self.seeds.hosts
|
|
264
|
-
self.whitelist = ScanWhitelist(*whitelist, strict_dns_scope=strict_scope
|
|
222
|
+
self.whitelist = ScanWhitelist(*whitelist, strict_dns_scope=strict_scope)
|
|
265
223
|
if blacklist is None:
|
|
266
224
|
blacklist = []
|
|
267
|
-
self.blacklist = ScanBlacklist(*blacklist
|
|
225
|
+
self.blacklist = ScanBlacklist(*blacklist)
|
|
268
226
|
|
|
269
227
|
@property
|
|
270
228
|
def json(self):
|
|
271
229
|
return {
|
|
272
|
-
"seeds": sorted(
|
|
273
|
-
"whitelist": sorted(
|
|
274
|
-
"blacklist": sorted(
|
|
230
|
+
"seeds": sorted(self.seeds.inputs),
|
|
231
|
+
"whitelist": sorted(self.whitelist.inputs),
|
|
232
|
+
"blacklist": sorted(self.blacklist.inputs),
|
|
275
233
|
"strict_scope": self.strict_scope,
|
|
276
234
|
"hash": self.hash.hex(),
|
|
277
235
|
"seed_hash": self.seeds.hash.hex(),
|
|
@@ -308,12 +266,9 @@ class BBOTTarget:
|
|
|
308
266
|
>>> preset.in_scope("http://www.evilcorp.com")
|
|
309
267
|
True
|
|
310
268
|
"""
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
return False
|
|
315
|
-
in_scope = e.scope_distance == 0 or self.whitelisted(e)
|
|
316
|
-
return in_scope and not self.blacklisted(e)
|
|
269
|
+
blacklisted = self.blacklisted(host)
|
|
270
|
+
whitelisted = self.whitelisted(host)
|
|
271
|
+
return whitelisted and not blacklisted
|
|
317
272
|
|
|
318
273
|
def blacklisted(self, host):
|
|
319
274
|
"""
|
|
@@ -347,18 +302,5 @@ class BBOTTarget:
|
|
|
347
302
|
"""
|
|
348
303
|
return host in self.whitelist
|
|
349
304
|
|
|
350
|
-
@property
|
|
351
|
-
def minimal(self):
|
|
352
|
-
"""
|
|
353
|
-
A slimmer, serializable version of the target designed for simple scope checks
|
|
354
|
-
|
|
355
|
-
This version doesn't have the events, only their hosts. This allows it to be passed across process boundaries.
|
|
356
|
-
"""
|
|
357
|
-
return self.__class__(
|
|
358
|
-
whitelist=self.whitelist.inputs,
|
|
359
|
-
blacklist=self.blacklist.inputs,
|
|
360
|
-
strict_scope=self.strict_scope,
|
|
361
|
-
)
|
|
362
|
-
|
|
363
305
|
def __eq__(self, other):
|
|
364
306
|
return self.hash == other.hash
|
|
@@ -591,7 +591,7 @@ def test_cli_module_validation(monkeypatch, caplog):
|
|
|
591
591
|
assert not caplog.text
|
|
592
592
|
monkeypatch.setattr("sys.argv", ["bbot", "-t", "asdf:::sdf"])
|
|
593
593
|
cli.main()
|
|
594
|
-
assert 'Unable to autodetect
|
|
594
|
+
assert 'Unable to autodetect data type from "asdf:::sdf"' in caplog.text
|
|
595
595
|
|
|
596
596
|
# incorrect flag
|
|
597
597
|
caplog.clear()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import ipaddress
|
|
3
|
+
from bbot.errors import ValidationError
|
|
4
|
+
from bbot.core.event.helpers import EventSeed
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_event_seeds():
|
|
8
|
+
# DNS_NAME
|
|
9
|
+
dns_seed = EventSeed("evilcOrp.com.")
|
|
10
|
+
assert dns_seed.type == "DNS_NAME"
|
|
11
|
+
assert dns_seed.data == "evilcorp.com"
|
|
12
|
+
assert dns_seed.host == "evilcorp.com"
|
|
13
|
+
assert dns_seed.input == "evilcorp.com"
|
|
14
|
+
assert dns_seed._target_type == "TARGET"
|
|
15
|
+
|
|
16
|
+
# IP_ADDRESS (IPv4)
|
|
17
|
+
ipv4_seed = EventSeed("192.168.1.1")
|
|
18
|
+
assert ipv4_seed.type == "IP_ADDRESS"
|
|
19
|
+
assert ipv4_seed.data == "192.168.1.1"
|
|
20
|
+
assert ipv4_seed.host == ipaddress.ip_address("192.168.1.1")
|
|
21
|
+
assert ipv4_seed.input == "192.168.1.1"
|
|
22
|
+
|
|
23
|
+
# Test various IPv6 formats
|
|
24
|
+
ipv6_formats = [
|
|
25
|
+
"2001:db8::ff00:42:8329", # Standard format
|
|
26
|
+
"2001:0db8:0000:0000:0000:ff00:0042:8329", # Full format
|
|
27
|
+
"2001:db8:0:0:0:ff00:42:8329", # Mixed format
|
|
28
|
+
"::1", # Loopback
|
|
29
|
+
"::ffff:192.168.1.1", # IPv4-mapped
|
|
30
|
+
"2001:db8::", # Subnet prefix
|
|
31
|
+
"fe80::1ff:fe23:4567:890a", # Link-local
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
# IP_ADDRESS (IPv6)
|
|
35
|
+
for ipv6 in ipv6_formats:
|
|
36
|
+
ipv6_seed = EventSeed(ipv6)
|
|
37
|
+
normalized_ipv6 = str(ipaddress.IPv6Address(ipv6))
|
|
38
|
+
assert ipv6_seed.type == "IP_ADDRESS"
|
|
39
|
+
assert ipv6_seed.data == normalized_ipv6
|
|
40
|
+
assert ipv6_seed.host == ipaddress.ip_address(ipv6)
|
|
41
|
+
assert ipv6_seed.input == normalized_ipv6
|
|
42
|
+
|
|
43
|
+
# IP_RANGE (IPv4)
|
|
44
|
+
ipv4_range_seed = EventSeed("192.168.1.1/24")
|
|
45
|
+
assert ipv4_range_seed.type == "IP_RANGE"
|
|
46
|
+
assert ipv4_range_seed.data == "192.168.1.0/24"
|
|
47
|
+
assert ipv4_range_seed.host == ipaddress.ip_network("192.168.1.0/24")
|
|
48
|
+
assert ipv4_range_seed.input == "192.168.1.0/24"
|
|
49
|
+
|
|
50
|
+
# IP_RANGE (IPv6)
|
|
51
|
+
ipv6_range_seed = EventSeed("2001:db8::ff00:42:8329/64")
|
|
52
|
+
assert ipv6_range_seed.type == "IP_RANGE"
|
|
53
|
+
assert ipv6_range_seed.data == "2001:db8::/64"
|
|
54
|
+
assert ipv6_range_seed.host == ipaddress.ip_network("2001:db8::/64")
|
|
55
|
+
assert ipv6_range_seed.input == "2001:db8::/64"
|
|
56
|
+
|
|
57
|
+
# OPEN_TCP_PORT (DNS)
|
|
58
|
+
open_port_dns_seed = EventSeed("evilcOrp.com:80")
|
|
59
|
+
assert open_port_dns_seed.type == "OPEN_TCP_PORT"
|
|
60
|
+
assert open_port_dns_seed.data == "evilcorp.com:80"
|
|
61
|
+
assert open_port_dns_seed.host == "evilcorp.com"
|
|
62
|
+
assert open_port_dns_seed.port == 80
|
|
63
|
+
assert open_port_dns_seed.input == "evilcorp.com:80"
|
|
64
|
+
|
|
65
|
+
# OPEN_TCP_PORT (IPv4)
|
|
66
|
+
open_port_ipv4_seed = EventSeed("192.168.1.1:80")
|
|
67
|
+
assert open_port_ipv4_seed.type == "OPEN_TCP_PORT"
|
|
68
|
+
assert open_port_ipv4_seed.data == "192.168.1.1:80"
|
|
69
|
+
assert open_port_ipv4_seed.host == ipaddress.ip_address("192.168.1.1")
|
|
70
|
+
assert open_port_ipv4_seed.port == 80
|
|
71
|
+
assert open_port_ipv4_seed.input == "192.168.1.1:80"
|
|
72
|
+
|
|
73
|
+
# OPEN_TCP_PORT (IPv6)
|
|
74
|
+
open_port_ipv6_seed = EventSeed("[2001:db8::42]:80")
|
|
75
|
+
assert open_port_ipv6_seed.type == "OPEN_TCP_PORT"
|
|
76
|
+
assert open_port_ipv6_seed.data == "[2001:db8::42]:80"
|
|
77
|
+
assert open_port_ipv6_seed.host == ipaddress.ip_address("2001:db8::42")
|
|
78
|
+
assert open_port_ipv6_seed.port == 80
|
|
79
|
+
assert open_port_ipv6_seed.input == "[2001:db8::42]:80"
|
|
80
|
+
|
|
81
|
+
# URL (DNS_NAME)
|
|
82
|
+
url_dns_seed = EventSeed("http://evilcOrp.com./index.html?a=b#c")
|
|
83
|
+
assert url_dns_seed.type == "URL_UNVERIFIED"
|
|
84
|
+
assert url_dns_seed.data == "http://evilcorp.com/index.html?a=b"
|
|
85
|
+
assert url_dns_seed.host == "evilcorp.com"
|
|
86
|
+
assert url_dns_seed.port == 80
|
|
87
|
+
assert url_dns_seed.input == "http://evilcorp.com/index.html?a=b"
|
|
88
|
+
|
|
89
|
+
# URL (IPv4)
|
|
90
|
+
url_ipv4_seed = EventSeed("https://192.168.1.1/index.html?a=b#c")
|
|
91
|
+
assert url_ipv4_seed.type == "URL_UNVERIFIED"
|
|
92
|
+
assert url_ipv4_seed.data == "https://192.168.1.1/index.html?a=b"
|
|
93
|
+
assert url_ipv4_seed.host == ipaddress.ip_address("192.168.1.1")
|
|
94
|
+
assert url_ipv4_seed.port == 443
|
|
95
|
+
assert url_ipv4_seed.input == "https://192.168.1.1/index.html?a=b"
|
|
96
|
+
|
|
97
|
+
# URL (IPv6)
|
|
98
|
+
url_ipv6_seed = EventSeed("https://[2001:db8::42]:8080/index.html?a=b#c")
|
|
99
|
+
assert url_ipv6_seed.type == "URL_UNVERIFIED"
|
|
100
|
+
assert url_ipv6_seed.data == "https://[2001:db8::42]:8080/index.html?a=b"
|
|
101
|
+
assert url_ipv6_seed.host == ipaddress.ip_address("2001:db8::42")
|
|
102
|
+
assert url_ipv6_seed.port == 8080
|
|
103
|
+
assert url_ipv6_seed.input == "https://[2001:db8::42]:8080/index.html?a=b"
|
|
104
|
+
|
|
105
|
+
# EMAIL_ADDRESS
|
|
106
|
+
email_seed = EventSeed("john.doe@evilcOrp.com")
|
|
107
|
+
assert email_seed.type == "EMAIL_ADDRESS"
|
|
108
|
+
assert email_seed.data == "john.doe@evilcorp.com"
|
|
109
|
+
assert email_seed.host == "evilcorp.com"
|
|
110
|
+
assert email_seed.port == None
|
|
111
|
+
assert email_seed.input == "john.doe@evilcorp.com"
|
|
112
|
+
|
|
113
|
+
email_seed_ipv4 = EventSeed("john.doe@192.168.1.1:80")
|
|
114
|
+
assert email_seed_ipv4.type == "EMAIL_ADDRESS"
|
|
115
|
+
assert email_seed_ipv4.data == "john.doe@192.168.1.1:80"
|
|
116
|
+
assert email_seed_ipv4.host == ipaddress.ip_address("192.168.1.1")
|
|
117
|
+
assert email_seed_ipv4.port == 80
|
|
118
|
+
assert email_seed_ipv4.input == "john.doe@192.168.1.1:80"
|
|
119
|
+
|
|
120
|
+
# ORG_STUB
|
|
121
|
+
org_stub_seed = EventSeed("ORG:evilcorp")
|
|
122
|
+
assert org_stub_seed.type == "ORG_STUB"
|
|
123
|
+
assert org_stub_seed.data == "evilcorp"
|
|
124
|
+
assert org_stub_seed.host == None
|
|
125
|
+
assert org_stub_seed.input == "ORG_STUB:evilcorp"
|
|
126
|
+
|
|
127
|
+
# USERNAME
|
|
128
|
+
username_seed = EventSeed("USER:john.doe")
|
|
129
|
+
assert username_seed.type == "USERNAME"
|
|
130
|
+
assert username_seed.data == "john.doe"
|
|
131
|
+
assert username_seed.host == None
|
|
132
|
+
assert username_seed.input == "USERNAME:john.doe"
|
|
133
|
+
|
|
134
|
+
# FILESYSTEM
|
|
135
|
+
filesystem_seed = EventSeed("FILE:/home/john/documents")
|
|
136
|
+
assert filesystem_seed.type == "FILESYSTEM"
|
|
137
|
+
assert filesystem_seed.data == {"path": "/home/john/documents"}
|
|
138
|
+
assert filesystem_seed.host == None
|
|
139
|
+
assert filesystem_seed.input == "FILESYSTEM:/home/john/documents"
|
|
140
|
+
|
|
141
|
+
# MOBILE_APP
|
|
142
|
+
mobile_app_seed = EventSeed("APK:https://play.google.com/store/apps/details?id=com.evilcorp.app")
|
|
143
|
+
assert mobile_app_seed.type == "MOBILE_APP"
|
|
144
|
+
assert mobile_app_seed.data == {"url": "https://play.google.com/store/apps/details?id=com.evilcorp.app"}
|
|
145
|
+
assert mobile_app_seed.host == None
|
|
146
|
+
assert mobile_app_seed.input == "MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app"
|
|
147
|
+
|
|
148
|
+
with pytest.raises(ValidationError):
|
|
149
|
+
EventSeed("INVALID:INVALID")
|
|
150
|
+
|
|
151
|
+
with pytest.raises(ValidationError):
|
|
152
|
+
EventSeed("^@#$^@#$")
|
|
153
|
+
|
|
154
|
+
# BLACKLIST_REGEX
|
|
155
|
+
blacklist_regex_seed = EventSeed("RE:evil[0-9]{3}")
|
|
156
|
+
assert blacklist_regex_seed.type == "BLACKLIST_REGEX"
|
|
157
|
+
assert blacklist_regex_seed.data == "evil[0-9]{3}"
|
|
158
|
+
assert blacklist_regex_seed.host == None
|
|
159
|
+
assert blacklist_regex_seed.input == "REGEX:evil[0-9]{3}"
|
|
160
|
+
assert blacklist_regex_seed._target_type == "BLACKLIST"
|
|
@@ -174,7 +174,7 @@ def test_preset_scope():
|
|
|
174
174
|
scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]}))
|
|
175
175
|
assert {str(h) for h in scan.preset.target.seeds.hosts} == {"1.2.3.4/32", "evilcorp.com"}
|
|
176
176
|
assert {e.data for e in scan.target.seeds} == {"1.2.3.4", "evilcorp.com"}
|
|
177
|
-
assert {e.data for e in scan.target.whitelist} == {"1.2.3.4", "evilcorp.com"}
|
|
177
|
+
assert {e.data for e in scan.target.whitelist} == {"1.2.3.4/32", "evilcorp.com"}
|
|
178
178
|
|
|
179
179
|
blank_preset = Preset()
|
|
180
180
|
blank_preset = blank_preset.bake()
|
|
@@ -272,13 +272,13 @@ def test_preset_scope():
|
|
|
272
272
|
}
|
|
273
273
|
assert preset_whitelist_baked.to_dict(include_target=True) == {
|
|
274
274
|
"target": ["evilcorp.org"],
|
|
275
|
-
"whitelist": ["1.2.3.
|
|
275
|
+
"whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
|
|
276
276
|
"blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
|
|
277
277
|
"config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}},
|
|
278
278
|
}
|
|
279
279
|
assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == {
|
|
280
280
|
"target": ["evilcorp.org"],
|
|
281
|
-
"whitelist": ["1.2.3.
|
|
281
|
+
"whitelist": ["1.2.3.0/24", "http://evilcorp.net/"],
|
|
282
282
|
"blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"],
|
|
283
283
|
"config": {"modules": {"secretsdb": {"otherthing": "asdf"}}},
|
|
284
284
|
}
|
|
@@ -87,7 +87,7 @@ def test_python_api_validation():
|
|
|
87
87
|
# invalid target
|
|
88
88
|
with pytest.raises(ValidationError) as error:
|
|
89
89
|
Scanner("asdf:::asdf")
|
|
90
|
-
assert str(error.value) == 'Unable to autodetect
|
|
90
|
+
assert str(error.value) == 'Unable to autodetect data type from "asdf:::asdf"'
|
|
91
91
|
# invalid module
|
|
92
92
|
with pytest.raises(ValidationError) as error:
|
|
93
93
|
Scanner(modules=["asdf"])
|