man-spider 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

man_spider/lib/__init__.py

@@ -1,4 +1,4 @@
-import parser
+from .parser import *
 from .util import *
 from .logger import *
-from .spider import *
+from .spider import *

man_spider/lib/errors.py

@@ -44,7 +44,6 @@ def impacket_error(e):
     '''
     Tries to format impacket exceptions nicely
     '''
-
     if type(e) in (SessionError, CSessionError):
         try:
             error_str = e.getErrorString()[0]
@@ -54,28 +53,3 @@ def impacket_error(e):
     if not e.args:
         e.args = ('',)
     return e
-
-
-def handle_impacket_error(e, smb_client, share='', filename='', display=False):
-    '''
-    Handle arbitrary Impacket errors
-    this is needed because the library doesn't implement proper inheritance for its exceptions
-    '''
-
-    resource_str = '/'.join([smb_client.server, share, filename]).rstrip('/')
-
-    if type(e) == KeyboardInterrupt:
-        raise
-    elif type(e) in (NetBIOSError, NetBIOSTimeout, BrokenPipeError, SessionError, CSessionError):
-        # the connection may need to be rebuilt
-        if type(e) in (SessionError, CSessionError):
-            if any([x in str(e) for x in ('PASSWORD_EXPIRED',)]):
-                smb_client.rebuild(e)
-        else:
-            smb_client.rebuild(e)
-    if type(e) in native_impacket_errors:
-        e = impacket_error(e)
-    if display:
-        log.debug(f'{resource_str}: {str(e)[:150]}')
-
-    return e

man_spider/lib/file.py

@@ -39,7 +39,7 @@ r    '''
             try:
                 smb_client.conn.getFile(self.share, self.name, f.write)
             except Exception as e:
-                handle_impacket_error(e, smb_client, self.share, self.name)
+                smb_client.handle_impacket_error(e, self.share, self.name)
                 raise FileRetrievalError(f'Error retrieving file "{str(self)}": {str(e)[:150]}')
 
         # reset cursor back to zero so .read() will return the whole file

man_spider/lib/parser/parser.py

@@ -1,32 +1,18 @@
 import re
 import magic
 import logging
-import textract
-from ..util import *
-from ..logger import *
+from time import sleep
 import subprocess as sp
-from ..logger import ColoredFormatter
+from extractous import Extractor
+
+from man_spider.lib.util import *
+from man_spider.lib.logger import *
 
 log = logging.getLogger('manspider.parser')
 
 
 class FileParser:
 
-    # parsed using the textract library
-    textract_extensions = [
-        '.doc',
-        '.docx',
-        '.xls',
-        '.xlsx',
-        '.ppt',
-        '.pptx',
-        '.pdf',
-        '.eml',
-        '.png',
-        '.jpg',
-        '.jpeg'
-    ]
-
     # don't parse files with these magic types
     magic_blacklist = [
         # PNG, JPEG, etc.
@@ -39,12 +25,11 @@ class FileParser:
 
 
     def __init__(self, filters, quiet=False):
-
         self.init_content_filters(filters)
+        self.extractor = Extractor()
         self.quiet = quiet
 
 
-
     def init_content_filters(self, file_content):
         '''
         Get ready to search by file content
@@ -91,7 +76,6 @@ class FileParser:
         return True
 
 
-
     def grep(self, content, pattern):
 
         if not self.quiet:
@@ -155,26 +139,18 @@ class FileParser:
 
         suffix = Path(str(file)).suffix.lower()
 
-        # textract
-        if suffix in self.textract_extensions:
-            binary_content = textract.process(str(file), encoding='utf-8')
-            text_content = better_decode(binary_content)
-
-        # normal
-        elif self.match_magic(file):
-            with open(str(file), 'rb') as f:
-                binary_content = f.read()
-            text_content = better_decode(binary_content)
-
-        else:
+        # blacklist certain mime types
+        if not self.match_magic(file):
             return matches
 
+        text_content, metadata = self.extractor.extract_file_to_string(str(file))
+
         # try to convert to UTF-8 for grep-friendliness
         try:
-            binary_content = text_content.encode('utf-8')
+            binary_content = text_content.encode('utf-8', errors='ignore')
         except Exception:
             pass
-            
+
         # count the matches
         for _filter, match in self.match(text_content):
             try:
@@ -188,4 +164,4 @@ class FileParser:
             if not self.quiet:
                 self.grep(binary_content, _filter.pattern)
 
-        return matches
+        return matches

man_spider/lib/smb.py

@@ -1,7 +1,7 @@
 import ntpath
-import struct
 import logging
 from .errors import *
+from contextlib import suppress
 from impacket.nmb import NetBIOSError, NetBIOSTimeout
 from impacket.smbconnection import SessionError, SMBConnection
 
@@ -14,7 +14,7 @@ class SMBClient:
     Wrapper around impacket's SMBConnection() object
     '''
 
-    def __init__(self, server, username, password, domain, nthash):
+    def __init__(self, server, username, password, domain, nthash, use_kerberos=False, aes_key="", dc_ip=None):
 
         self.server = server
 
@@ -24,27 +24,89 @@ class SMBClient:
         self.password = password
         self.domain = domain
         self.nthash = nthash
+        self.use_kerberos = use_kerberos
+        self.aes_key = aes_key
+        self.dc_ip = dc_ip
+        self.hostname = None
+        self.dns_domain = None
         if self.nthash:
             self.lmhash = 'aad3b435b51404eeaad3b435b51404ee'
         else:
             self.lmhash = ''
+        self._shares = None
+
+
+    def list_shares(self):
+        '''
+        List shares on the SMB server
+        '''
+        resp = self.conn.listShares()
+        for i in range(len(resp)):
+            sharename = resp[i]['shi1_netname'][:-1]
+            log.debug(f'{self.server}: Found share: {sharename}')
+            yield sharename
 
 
     @property
     def shares(self):
-
+        if self._shares is None:
+            try:
+                self._shares = list(self.list_shares())
+            except Exception as e:
+                e = self.handle_impacket_error(e)
+                log.debug(f'{self.server}: Error listing shares: {e}, retrying...')
+                self.rebuild(e)
+                try:
+                    self._shares = list(self.list_shares())
+                except Exception as e:
+                    e = self.handle_impacket_error(e)
+                    log.warning(f'{self.server}: Error listing shares: {e}')
+                    self.rebuild(e)
+        return self._shares or []
+
+
+    def get_hostname(self):
+        '''
+        Get the hostname from the SMB connection
+        '''
         try:
-            resp = self.conn.listShares()
-            for i in range(len(resp)):
-                sharename = resp[i]['shi1_netname'][:-1]
-                log.debug(f'{self.server}: Found share: {sharename}')
-                yield sharename
-            
+            conn = SMBConnection(
+                self.server,
+                self.server,
+                None,
+                445,
+                timeout=10,
+            )
+            with suppress(Exception):
+                conn.login("", "")
+
+            if self.hostname is None:
+                try:
+                    # Get the server name from SMB
+                    self.hostname = str(conn.getServerName()).strip().replace("\x00", "").lower()
+                    if self.hostname:
+                        log.debug(f'{self.server}: Got hostname: {self.hostname}')
+                    else:
+                        log.debug(f'{self.server}: No hostname found')
+                except Exception as e:
+                    log.debug(f'{self.server}: Error getting hostname from SMB: {e}')
+                    self.hostname = ""
+
+            if self.dns_domain is None:
+                try:
+                    self.dns_domain = str(conn.getServerDNSDomainName()).strip().replace("\x00", "").lower()
+                    if self.dns_domain:
+                        log.debug(f'{self.server}: Got DNS domain: {self.dns_domain}')
+                    else:
+                        log.debug(f'{self.server}: No DNS domain found')
+                except Exception as e:
+                    log.debug(f'{self.server}: Error getting DNS domain: {e}')
+                    self.dns_domain = (self.domain if self.domain else "")
+
         except Exception as e:
-            e = handle_impacket_error(e, self)
-            log.warning(f'{self.server}: Error listing shares: {e}')
-            
+            log.debug(f'{self.server}: Error getting hostname: {e}')
 
+        return self.hostname, self.domain
 
     def login(self, refresh=False, first_try=True):
         '''
@@ -54,9 +116,17 @@ class SMBClient:
         Return False if logon failed
         '''
 
+        target_server = self.server
+        if self.use_kerberos:
+            hostname, domain = self.get_hostname()
+            if hostname:
+                target_server = hostname
+                if domain:
+                    target_server = f"{hostname}.{domain}"
+
         if self.conn is None or refresh:
             try:
-                self.conn = SMBConnection(self.server, self.server, sess_port=445, timeout=20)
+                self.conn = SMBConnection(target_server, target_server, sess_port=445, timeout=20)
             except Exception as e:
                 log.debug(impacket_error(e))
                 return None
@@ -67,10 +137,23 @@ class SMBClient:
                     # skip to guest / null session
                     assert False
 
-                log.debug(f'{self.server}: Authenticating as "{self.domain}\\{self.username}"')
+                user_str = self.username
+                if self.domain:
+                    user_str = f'{self.domain}\\{self.username}'
+                log.debug(f'{target_server} ({self.server}): Authenticating as "{user_str}"')
 
+                if self.use_kerberos:
+                    self.conn.kerberosLogin(
+                        self.username,
+                        self.password,
+                        self.domain,
+                        self.lmhash,
+                        self.nthash,
+                        self.aes_key,
+                        kdcHost=self.dc_ip,
+                    )
                 # pass the hash if requested
-                if self.nthash and not self.password:
+                elif self.nthash and not self.password:
                     self.conn.login(
                         self.username,
                         '',
@@ -92,7 +175,7 @@ class SMBClient:
             except Exception as e:
 
                 if type(e) != AssertionError:
-                    e = handle_impacket_error(e, self, display=True)
+                    e = self.handle_impacket_error(e, display=True)
 
                 # try guest account, then null session if logon failed
                 if first_try:
@@ -136,15 +219,37 @@ class SMBClient:
                 if f.get_longname() not in ['', '.', '..']:
                     yield f
         except Exception as e:
-            e = handle_impacket_error(e, self)
+            e = self.handle_impacket_error(e)
             raise FileListError(f'{e.args}: Error listing files at "{share}{nt_path}"')
 
 
-
     def rebuild(self, error=''):
         '''
         Rebuild our SMBConnection() if it gets borked
         '''
-
         log.debug(f'Rebuilding connection to {self.server} after error: {error}')
-        self.login(refresh=True)
+        self.login(refresh=True)
+
+
+    def handle_impacket_error(self, e, share='', filename='', display=False):
+        '''
+        Handle arbitrary Impacket errors
+        this is needed because the library doesn't implement proper inheritance for its exceptions
+        '''
+        resource_str = '/'.join([self.server, share, filename]).rstrip('/')
+
+        if type(e) == KeyboardInterrupt:
+            raise
+        elif type(e) in (NetBIOSError, NetBIOSTimeout, BrokenPipeError, SessionError, CSessionError):
+            # the connection may need to be rebuilt
+            if type(e) in (SessionError, CSessionError):
+                if any([x in str(e) for x in ('PASSWORD_EXPIRED',)]):
+                    self.rebuild(e)
+            else:
+                self.rebuild(e)
+        if type(e) in native_impacket_errors:
+            e = impacket_error(e)
+        if display:
+            log.debug(f'{resource_str}: {str(e)[:150]}')
+
+        return e

man_spider/lib/spider.py

@@ -25,6 +25,9 @@ class MANSPIDER:
         self.password           = options.password
         self.domain             = options.domain
         self.nthash             = options.hash
+        self.use_kerberos       = options.kerberos
+        self.aes_key            = options.aes_key
+        self.dc_ip              = options.dc_ip
         self.max_failed_logons  = options.max_failed_logons
         self.max_filesize       = options.max_filesize
 
@@ -64,6 +67,10 @@ class MANSPIDER:
 
         # directory to store matching documents
         self.loot_dir = Path.home() / '.manspider' / 'loot'
+
+        if(options.loot_dir):
+            self.loot_dir=Path(options.loot_dir)
+        
         self.loot_dir.mkdir(parents=True, exist_ok=True)
 
         if not options.no_download:
@@ -200,10 +207,13 @@ class MANSPIDER:
                 self.password,
                 self.domain,
                 self.nthash,
+                self.use_kerberos,
+                self.aes_key,
+                self.dc_ip
             )
             logon_result = smb_client.login()
             if logon_result == False:
                 self.failed_logons += 1
             self.smb_client_cache[target] = smb_client
 
-        return smb_client
+        return smb_client

man_spider/lib/spiderling.py

@@ -67,13 +67,16 @@ class Spiderling:
 
             else:
                 self.local = False
-                
+
                 self.smb_client = SMBClient(
                     target,
                     parent.username,
                     parent.password,
                     parent.domain,
                     parent.nthash,
+                    parent.use_kerberos,
+                    parent.aes_key,
+                    parent.dc_ip
                 )
 
                 logon_result = self.smb_client.login()
@@ -157,7 +160,7 @@ class Spiderling:
         else:
             for share in self.shares:
                 for remote_file in self.list_files(share):
-                    if not self.parent.no_download:
+                    if not self.parent.no_download or self.parent.parser.content_filters:
                         self.get_file(remote_file)
                     yield remote_file
 
@@ -260,7 +263,7 @@ class Spiderling:
                     try:
                         filesize = f.get_filesize()
                     except Exception as e:
-                        handle_impacket_error(e)
+                        self.smb_client.handle_impacket_error(e)
                         continue
 
                     # make the RemoteFile object (the file won't be read yet)
@@ -269,7 +272,7 @@ class Spiderling:
 
                     # if it's a non-empty file that's smaller than the size limit
                     if filesize > 0 and filesize < self.parent.max_filesize:
-                        
+
                         # if it matched filename/extension filters and we're downloading files
                         if (self.parent.file_extensions or self.parent.filename_filters) and not self.parent.no_download:
                             # but the extension is marked as "don't parse"
@@ -455,4 +458,5 @@ class Spiderling:
         except FileRetrievalError as e:
             log.debug(f'{self.target}: {e}')
 
-        return False
+        return False
+

man_spider/manspider.py

@@ -83,8 +83,12 @@ def main():
     parser.add_argument('-u', '--username',     default='',                 help='username for authentication')
     parser.add_argument('-p', '--password',     default='',                 help='password for authentication')
     parser.add_argument('-d', '--domain',       default='',                 help='domain for authentication')
+    parser.add_argument('-l','--loot-dir',      default='',                 help='loot directory (default ~/.manspider/)')
     parser.add_argument('-m', '--maxdepth',     type=int,   default=10,     help='maximum depth to spider (default: 10)')
     parser.add_argument('-H', '--hash',         default='',                 help='NTLM hash for authentication')
+    parser.add_argument('-k', '--kerberos',     action='store_true',       help='Use Kerberos authentication. Grabs credentials from ccache file (KRB5CCNAME) based on target parameters')
+    parser.add_argument('-aesKey', '--aes-key', action='store', metavar='HEX', help='AES key to use for Kerberos Authentication (128 or 256 bits)')
+    parser.add_argument('-dc-ip', '--dc-ip',    action='store', metavar='IP', help='IP Address of the domain controller. If omitted it will use the domain part (FQDN) specified in the target parameter')
     parser.add_argument('-t', '--threads',      type=int,   default=5,      help='concurrent threads (default: 5)')
     parser.add_argument('-f', '--filenames', nargs='+', default=[],         help=f'filter filenames using regex (space-separated)', metavar='REGEX')
     parser.add_argument('-e', '--extensions',nargs='+', default=[],         help='only show filenames with these extensions (space-separated, e.g. `docx xlsx` for only word & excel docs)', metavar='EXT')
@@ -100,6 +104,7 @@ def main():
     parser.add_argument('-o', '--or-logic', action='store_true',            help=f'use OR logic instead of AND (files are downloaded if filename OR extension OR content match)')
     parser.add_argument('-s', '--max-filesize', type=human_to_int, default=human_to_int('10M'), help=f'don\'t retrieve files over this size, e.g. "500K" or ".5M" (default: 10M)', metavar='SIZE')
     parser.add_argument('-v', '--verbose', action='store_true',             help='show debugging messages')
+    
 
     syntax_error = False
     try:
@@ -113,6 +118,10 @@ def main():
         if options.verbose:
             log.setLevel('DEBUG')
 
+        if options.kerberos and not "KRB5CCNAME" in os.environ:
+            log.error("KRB5CCNAME is not set in the environment")
+            sys.exit(1)
+
         # make sure extension formats are valid
         for i, extension in enumerate(options.extensions):
             if extension and not extension.startswith('.'):