man-spider 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. man_spider/__init__.py +0 -0
  2. man_spider/lib/errors.py +0 -26
  3. man_spider/lib/file.py +1 -1
  4. man_spider/lib/parser/parser.py +13 -37
  5. man_spider/lib/smb.py +125 -20
  6. man_spider/lib/spider.py +6 -0
  7. man_spider/lib/spiderling.py +4 -1
  8. man_spider/manspider.py +7 -0
  9. man_spider-1.1.0.dist-info/LICENSE +674 -0
  10. {man_spider-1.0.4.dist-info → man_spider-1.1.0.dist-info}/METADATA +11 -10
  11. man_spider-1.1.0.dist-info/RECORD +18 -0
  12. {man_spider-1.0.4.dist-info → man_spider-1.1.0.dist-info}/WHEEL +1 -1
  13. man_spider/logs/manspider_05-17-2021.log +0 -2070
  14. man_spider/loot/share.blacklanternsecurity.com_Share_BLS_GVT_Booz_Endgame 3.9.0 ovf_README.txt +0 -14
  15. man_spider/loot/share.blacklanternsecurity.com_Share_RA Retrospective_20190626_BLS_RA_Retrospective_Notes_v001.docx +0 -0
  16. man_spider/loot/share.blacklanternsecurity.com_Share_RA Retrospective_20190626_BLS_RA_Retrospective_Notes_v001.pdf +0 -0
  17. man_spider/loot/share.blacklanternsecurity.com_Share_Research_bls_app_01_references_Botconf2016_TomUeltschi_Sysmon.pdf +0 -0
  18. man_spider/loot/share.blacklanternsecurity.com_Share_Research_bls_app_01_references_htaw05tracking_hackers_on_your_network_with_sysinternals_sysmon.pdf +0 -0
  19. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_hipaaIntegrity_20170221_hipaaIntegrity_original_HIPAA_Integrity_Safeguard_Forms_v4.1.docx +0 -0
  20. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_hipaaIntegrity_PP_analysis_20170220_PR_BN_SR_List_v001.xlsx +0 -0
  21. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_hipaaIntegrity_PP_analysis_HIPAA_Integrity_SR_PP_v4.1.docx +0 -0
  22. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_audit_logging_information_logging_standard.docx +0 -0
  23. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_audit_logging_nistspecialpublication80092.pdf +0 -0
  24. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_20170221_BG_IR_v001.docx +0 -0
  25. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_20170221_BG_IR_v002.docx +0 -0
  26. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_20170221_BG_IR_v003.docx +0 -0
  27. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_20170221_BG_IR_v004.docx +0 -0
  28. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_20170221_BG_IR_v005.docx +0 -0
  29. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_NIST.SP.80061r2.pdf +0 -0
  30. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_data_breach_response.docx +0 -0
  31. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_policy_creation_incident_response_eventmonitoringincidentresponse34232.pdf +0 -0
  32. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_sans_docs_riskanalysishipaacompliancy1554.pdf +0 -0
  33. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_HIPAA_sans_docs_riskanalysishipaacompliancy15542.pdf +0 -0
  34. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_NIST_NIST.SP.80053r4.pdf +0 -0
  35. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_NIST_nistspecialpublication80030r1.pdf +0 -0
  36. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_pci_dss_20160809_pcidss_req.xlsx +0 -0
  37. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_pci_dss_PCI_DSS_v32.pdf +0 -0
  38. man_spider/loot/share.blacklanternsecurity.com_Share_Research_regulatory_pci_dss_SAQ_D_v3_Merchant.pdf +0 -0
  39. man_spider/loot/share.blacklanternsecurity.com_Share_Software_Adobe_Download.txt +0 -6
  40. man_spider/loot/share.blacklanternsecurity.com_Share_Software_Tools_RubberDucky_Flashing ducky hak5darren_USBRubberDucky Wiki GitHub.pdf +1 -1141
  41. man_spider/loot/share.blacklanternsecurity.com_Share_Software_Vulnerable_Software_Oracle_CVE20165663_4_5_ RCE and Cardholder Data Exfiltration in Oracles Hotel Management Platform 126Kr.pdf +0 -0
  42. man_spider-1.0.4.dist-info/RECORD +0 -45
  43. {man_spider-1.0.4.dist-info → man_spider-1.1.0.dist-info}/entry_points.txt +0 -0
man_spider/__init__.py ADDED
File without changes
man_spider/lib/errors.py CHANGED
@@ -44,7 +44,6 @@ def impacket_error(e):
44
44
  '''
45
45
  Tries to format impacket exceptions nicely
46
46
  '''
47
-
48
47
  if type(e) in (SessionError, CSessionError):
49
48
  try:
50
49
  error_str = e.getErrorString()[0]
@@ -54,28 +53,3 @@ def impacket_error(e):
54
53
  if not e.args:
55
54
  e.args = ('',)
56
55
  return e
57
-
58
-
59
- def handle_impacket_error(e, smb_client, share='', filename='', display=False):
60
- '''
61
- Handle arbitrary Impacket errors
62
- this is needed because the library doesn't implement proper inheritance for its exceptions
63
- '''
64
-
65
- resource_str = '/'.join([smb_client.server, share, filename]).rstrip('/')
66
-
67
- if type(e) == KeyboardInterrupt:
68
- raise
69
- elif type(e) in (NetBIOSError, NetBIOSTimeout, BrokenPipeError, SessionError, CSessionError):
70
- # the connection may need to be rebuilt
71
- if type(e) in (SessionError, CSessionError):
72
- if any([x in str(e) for x in ('PASSWORD_EXPIRED',)]):
73
- smb_client.rebuild(e)
74
- else:
75
- smb_client.rebuild(e)
76
- if type(e) in native_impacket_errors:
77
- e = impacket_error(e)
78
- if display:
79
- log.debug(f'{resource_str}: {str(e)[:150]}')
80
-
81
- return e
man_spider/lib/file.py CHANGED
@@ -39,7 +39,7 @@ r '''
39
39
  try:
40
40
  smb_client.conn.getFile(self.share, self.name, f.write)
41
41
  except Exception as e:
42
- handle_impacket_error(e, smb_client, self.share, self.name)
42
+ smb_client.handle_impacket_error(e, self.share, self.name)
43
43
  raise FileRetrievalError(f'Error retrieving file "{str(self)}": {str(e)[:150]}')
44
44
 
45
45
  # reset cursor back to zero so .read() will return the whole file
@@ -1,32 +1,18 @@
1
1
  import re
2
2
  import magic
3
3
  import logging
4
- import textract
5
- from ..util import *
6
- from ..logger import *
4
+ from time import sleep
7
5
  import subprocess as sp
8
- from ..logger import ColoredFormatter
6
+ from extractous import Extractor
7
+
8
+ from man_spider.lib.util import *
9
+ from man_spider.lib.logger import *
9
10
 
10
11
  log = logging.getLogger('manspider.parser')
11
12
 
12
13
 
13
14
  class FileParser:
14
15
 
15
- # parsed using the textract library
16
- textract_extensions = [
17
- '.doc',
18
- '.docx',
19
- '.xls',
20
- '.xlsx',
21
- '.ppt',
22
- '.pptx',
23
- '.pdf',
24
- '.eml',
25
- '.png',
26
- '.jpg',
27
- '.jpeg'
28
- ]
29
-
30
16
  # don't parse files with these magic types
31
17
  magic_blacklist = [
32
18
  # PNG, JPEG, etc.
@@ -39,12 +25,11 @@ class FileParser:
39
25
 
40
26
 
41
27
  def __init__(self, filters, quiet=False):
42
-
43
28
  self.init_content_filters(filters)
29
+ self.extractor = Extractor()
44
30
  self.quiet = quiet
45
31
 
46
32
 
47
-
48
33
  def init_content_filters(self, file_content):
49
34
  '''
50
35
  Get ready to search by file content
@@ -91,7 +76,6 @@ class FileParser:
91
76
  return True
92
77
 
93
78
 
94
-
95
79
  def grep(self, content, pattern):
96
80
 
97
81
  if not self.quiet:
@@ -155,26 +139,18 @@ class FileParser:
155
139
 
156
140
  suffix = Path(str(file)).suffix.lower()
157
141
 
158
- # textract
159
- if suffix in self.textract_extensions:
160
- binary_content = textract.process(str(file), encoding='utf-8')
161
- text_content = better_decode(binary_content)
162
-
163
- # normal
164
- elif self.match_magic(file):
165
- with open(str(file), 'rb') as f:
166
- binary_content = f.read()
167
- text_content = better_decode(binary_content)
168
-
169
- else:
142
+ # blacklist certain mime types
143
+ if not self.match_magic(file):
170
144
  return matches
171
145
 
146
+ text_content, metadata = self.extractor.extract_file_to_string(str(file))
147
+
172
148
  # try to convert to UTF-8 for grep-friendliness
173
149
  try:
174
- binary_content = text_content.encode('utf-8')
150
+ binary_content = text_content.encode('utf-8', errors='ignore')
175
151
  except Exception:
176
152
  pass
177
-
153
+
178
154
  # count the matches
179
155
  for _filter, match in self.match(text_content):
180
156
  try:
@@ -188,4 +164,4 @@ class FileParser:
188
164
  if not self.quiet:
189
165
  self.grep(binary_content, _filter.pattern)
190
166
 
191
- return matches
167
+ return matches
man_spider/lib/smb.py CHANGED
@@ -1,7 +1,7 @@
1
1
  import ntpath
2
- import struct
3
2
  import logging
4
3
  from .errors import *
4
+ from contextlib import suppress
5
5
  from impacket.nmb import NetBIOSError, NetBIOSTimeout
6
6
  from impacket.smbconnection import SessionError, SMBConnection
7
7
 
@@ -14,7 +14,7 @@ class SMBClient:
14
14
  Wrapper around impacket's SMBConnection() object
15
15
  '''
16
16
 
17
- def __init__(self, server, username, password, domain, nthash):
17
+ def __init__(self, server, username, password, domain, nthash, use_kerberos=False, aes_key="", dc_ip=None):
18
18
 
19
19
  self.server = server
20
20
 
@@ -24,27 +24,89 @@ class SMBClient:
24
24
  self.password = password
25
25
  self.domain = domain
26
26
  self.nthash = nthash
27
+ self.use_kerberos = use_kerberos
28
+ self.aes_key = aes_key
29
+ self.dc_ip = dc_ip
30
+ self.hostname = None
31
+ self.dns_domain = None
27
32
  if self.nthash:
28
33
  self.lmhash = 'aad3b435b51404eeaad3b435b51404ee'
29
34
  else:
30
35
  self.lmhash = ''
36
+ self._shares = None
37
+
38
+
39
+ def list_shares(self):
40
+ '''
41
+ List shares on the SMB server
42
+ '''
43
+ resp = self.conn.listShares()
44
+ for i in range(len(resp)):
45
+ sharename = resp[i]['shi1_netname'][:-1]
46
+ log.debug(f'{self.server}: Found share: {sharename}')
47
+ yield sharename
31
48
 
32
49
 
33
50
  @property
34
51
  def shares(self):
35
-
52
+ if self._shares is None:
53
+ try:
54
+ self._shares = list(self.list_shares())
55
+ except Exception as e:
56
+ e = self.handle_impacket_error(e)
57
+ log.debug(f'{self.server}: Error listing shares: {e}, retrying...')
58
+ self.rebuild(e)
59
+ try:
60
+ self._shares = list(self.list_shares())
61
+ except Exception as e:
62
+ e = self.handle_impacket_error(e)
63
+ log.warning(f'{self.server}: Error listing shares: {e}')
64
+ self.rebuild(e)
65
+ return self._shares or []
66
+
67
+
68
+ def get_hostname(self):
69
+ '''
70
+ Get the hostname from the SMB connection
71
+ '''
36
72
  try:
37
- resp = self.conn.listShares()
38
- for i in range(len(resp)):
39
- sharename = resp[i]['shi1_netname'][:-1]
40
- log.debug(f'{self.server}: Found share: {sharename}')
41
- yield sharename
42
-
73
+ conn = SMBConnection(
74
+ self.server,
75
+ self.server,
76
+ None,
77
+ 445,
78
+ timeout=10,
79
+ )
80
+ with suppress(Exception):
81
+ conn.login("", "")
82
+
83
+ if self.hostname is None:
84
+ try:
85
+ # Get the server name from SMB
86
+ self.hostname = str(conn.getServerName()).strip().replace("\x00", "").lower()
87
+ if self.hostname:
88
+ log.debug(f'{self.server}: Got hostname: {self.hostname}')
89
+ else:
90
+ log.debug(f'{self.server}: No hostname found')
91
+ except Exception as e:
92
+ log.debug(f'{self.server}: Error getting hostname from SMB: {e}')
93
+ self.hostname = ""
94
+
95
+ if self.dns_domain is None:
96
+ try:
97
+ self.dns_domain = str(conn.getServerDNSDomainName()).strip().replace("\x00", "").lower()
98
+ if self.dns_domain:
99
+ log.debug(f'{self.server}: Got DNS domain: {self.dns_domain}')
100
+ else:
101
+ log.debug(f'{self.server}: No DNS domain found')
102
+ except Exception as e:
103
+ log.debug(f'{self.server}: Error getting DNS domain: {e}')
104
+ self.dns_domain = (self.domain if self.domain else "")
105
+
43
106
  except Exception as e:
44
- e = handle_impacket_error(e, self)
45
- log.warning(f'{self.server}: Error listing shares: {e}')
46
-
107
+ log.debug(f'{self.server}: Error getting hostname: {e}')
47
108
 
109
+ return self.hostname, self.domain
48
110
 
49
111
  def login(self, refresh=False, first_try=True):
50
112
  '''
@@ -54,9 +116,17 @@ class SMBClient:
54
116
  Return False if logon failed
55
117
  '''
56
118
 
119
+ target_server = self.server
120
+ if self.use_kerberos:
121
+ hostname, domain = self.get_hostname()
122
+ if hostname:
123
+ target_server = hostname
124
+ if domain:
125
+ target_server = f"{hostname}.{domain}"
126
+
57
127
  if self.conn is None or refresh:
58
128
  try:
59
- self.conn = SMBConnection(self.server, self.server, sess_port=445, timeout=20)
129
+ self.conn = SMBConnection(target_server, target_server, sess_port=445, timeout=20)
60
130
  except Exception as e:
61
131
  log.debug(impacket_error(e))
62
132
  return None
@@ -67,10 +137,23 @@ class SMBClient:
67
137
  # skip to guest / null session
68
138
  assert False
69
139
 
70
- log.debug(f'{self.server}: Authenticating as "{self.domain}\\{self.username}"')
140
+ user_str = self.username
141
+ if self.domain:
142
+ user_str = f'{self.domain}\\{self.username}'
143
+ log.debug(f'{target_server} ({self.server}): Authenticating as "{user_str}"')
71
144
 
145
+ if self.use_kerberos:
146
+ self.conn.kerberosLogin(
147
+ self.username,
148
+ self.password,
149
+ self.domain,
150
+ self.lmhash,
151
+ self.nthash,
152
+ self.aes_key,
153
+ kdcHost=self.dc_ip,
154
+ )
72
155
  # pass the hash if requested
73
- if self.nthash and not self.password:
156
+ elif self.nthash and not self.password:
74
157
  self.conn.login(
75
158
  self.username,
76
159
  '',
@@ -92,7 +175,7 @@ class SMBClient:
92
175
  except Exception as e:
93
176
 
94
177
  if type(e) != AssertionError:
95
- e = handle_impacket_error(e, self, display=True)
178
+ e = self.handle_impacket_error(e, display=True)
96
179
 
97
180
  # try guest account, then null session if logon failed
98
181
  if first_try:
@@ -136,15 +219,37 @@ class SMBClient:
136
219
  if f.get_longname() not in ['', '.', '..']:
137
220
  yield f
138
221
  except Exception as e:
139
- e = handle_impacket_error(e, self)
222
+ e = self.handle_impacket_error(e)
140
223
  raise FileListError(f'{e.args}: Error listing files at "{share}{nt_path}"')
141
224
 
142
225
 
143
-
144
226
  def rebuild(self, error=''):
145
227
  '''
146
228
  Rebuild our SMBConnection() if it gets borked
147
229
  '''
148
-
149
230
  log.debug(f'Rebuilding connection to {self.server} after error: {error}')
150
- self.login(refresh=True)
231
+ self.login(refresh=True)
232
+
233
+
234
+ def handle_impacket_error(self, e, share='', filename='', display=False):
235
+ '''
236
+ Handle arbitrary Impacket errors
237
+ this is needed because the library doesn't implement proper inheritance for its exceptions
238
+ '''
239
+ resource_str = '/'.join([self.server, share, filename]).rstrip('/')
240
+
241
+ if type(e) == KeyboardInterrupt:
242
+ raise
243
+ elif type(e) in (NetBIOSError, NetBIOSTimeout, BrokenPipeError, SessionError, CSessionError):
244
+ # the connection may need to be rebuilt
245
+ if type(e) in (SessionError, CSessionError):
246
+ if any([x in str(e) for x in ('PASSWORD_EXPIRED',)]):
247
+ self.rebuild(e)
248
+ else:
249
+ self.rebuild(e)
250
+ if type(e) in native_impacket_errors:
251
+ e = impacket_error(e)
252
+ if display:
253
+ log.debug(f'{resource_str}: {str(e)[:150]}')
254
+
255
+ return e
man_spider/lib/spider.py CHANGED
@@ -25,6 +25,9 @@ class MANSPIDER:
25
25
  self.password = options.password
26
26
  self.domain = options.domain
27
27
  self.nthash = options.hash
28
+ self.use_kerberos = options.kerberos
29
+ self.aes_key = options.aes_key
30
+ self.dc_ip = options.dc_ip
28
31
  self.max_failed_logons = options.max_failed_logons
29
32
  self.max_filesize = options.max_filesize
30
33
 
@@ -204,6 +207,9 @@ class MANSPIDER:
204
207
  self.password,
205
208
  self.domain,
206
209
  self.nthash,
210
+ self.use_kerberos,
211
+ self.aes_key,
212
+ self.dc_ip
207
213
  )
208
214
  logon_result = smb_client.login()
209
215
  if logon_result == False:
@@ -74,6 +74,9 @@ class Spiderling:
74
74
  parent.password,
75
75
  parent.domain,
76
76
  parent.nthash,
77
+ parent.use_kerberos,
78
+ parent.aes_key,
79
+ parent.dc_ip
77
80
  )
78
81
 
79
82
  logon_result = self.smb_client.login()
@@ -260,7 +263,7 @@ class Spiderling:
260
263
  try:
261
264
  filesize = f.get_filesize()
262
265
  except Exception as e:
263
- handle_impacket_error(e)
266
+ self.smb_client.handle_impacket_error(e)
264
267
  continue
265
268
 
266
269
  # make the RemoteFile object (the file won't be read yet)
man_spider/manspider.py CHANGED
@@ -86,6 +86,9 @@ def main():
86
86
  parser.add_argument('-l','--loot-dir', default='', help='loot directory (default ~/.manspider/)')
87
87
  parser.add_argument('-m', '--maxdepth', type=int, default=10, help='maximum depth to spider (default: 10)')
88
88
  parser.add_argument('-H', '--hash', default='', help='NTLM hash for authentication')
89
+ parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos authentication. Grabs credentials from ccache file (KRB5CCNAME) based on target parameters')
90
+ parser.add_argument('-aesKey', '--aes-key', action='store', metavar='HEX', help='AES key to use for Kerberos Authentication (128 or 256 bits)')
91
+ parser.add_argument('-dc-ip', '--dc-ip', action='store', metavar='IP', help='IP Address of the domain controller. If omitted it will use the domain part (FQDN) specified in the target parameter')
89
92
  parser.add_argument('-t', '--threads', type=int, default=5, help='concurrent threads (default: 5)')
90
93
  parser.add_argument('-f', '--filenames', nargs='+', default=[], help=f'filter filenames using regex (space-separated)', metavar='REGEX')
91
94
  parser.add_argument('-e', '--extensions',nargs='+', default=[], help='only show filenames with these extensions (space-separated, e.g. `docx xlsx` for only word & excel docs)', metavar='EXT')
@@ -115,6 +118,10 @@ def main():
115
118
  if options.verbose:
116
119
  log.setLevel('DEBUG')
117
120
 
121
+ if options.kerberos and not "KRB5CCNAME" in os.environ:
122
+ log.error("KRB5CCNAME is not set in the environment")
123
+ sys.exit(1)
124
+
118
125
  # make sure extension formats are valid
119
126
  for i, extension in enumerate(options.extensions):
120
127
  if extension and not extension.startswith('.'):