man-spider 1.1.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

man_spider/lib/errors.py

@@ -5,21 +5,25 @@ from impacket.smb import SessionError, UnsupportedFeature
 from impacket.smbconnection import SessionError as CSessionError
 
 # set up logging
-log = logging.getLogger('manspider')
+log = logging.getLogger("manspider")
 
 
 class MANSPIDERError(Exception):
     pass
 
+
 class FileRetrievalError(MANSPIDERError):
     pass
 
+
 class ShareListError(MANSPIDERError):
     pass
 
+
 class FileListError(MANSPIDERError):
     pass
 
+
 class LogonFailure(MANSPIDERError):
     pass
 
@@ -41,9 +45,9 @@ impacket_errors = (
 
 
 def impacket_error(e):
-    '''
+    """
     Tries to format impacket exceptions nicely
-    '''
+    """
     if type(e) in (SessionError, CSessionError):
         try:
             error_str = e.getErrorString()[0]
@@ -51,5 +55,5 @@ def impacket_error(e):
         except (IndexError,):
             pass
     if not e.args:
-        e.args = ('',)
+        e.args = ("",)
     return e

man_spider/lib/file.py

@@ -1,14 +1,14 @@
-import io
-from .util import *
-from .errors import *
 from pathlib import Path
 
+from man_spider.lib.util import *
+from man_spider.lib.errors import *
 
-class RemoteFile():
-    '''
-    Represents a file on an SMB share
-    Passed from a spiderling up to its parent spide
-r    '''
+
+class RemoteFile:
+    """
+        Represents a file on an SMB share
+        Passed from a spiderling up to its parent spide
+    r"""
 
     def __init__(self, name, share, target, size=0):
 
@@ -19,23 +19,21 @@ r    '''
         self.smb_client = None
 
         file_suffix = Path(name).suffix.lower()
-        self.tmp_filename = Path('/tmp/.manspider') / (random_string(15) + file_suffix)
-
+        self.tmp_filename = Path("/tmp/.manspider") / (random_string(15) + file_suffix)
 
     def get(self, smb_client=None):
-        '''
+        """
         Downloads file to self.tmp_filename
 
         NOTE: SMBConnection() can't be passed through a multiprocessing queue
               This means that smb_client must be set after the file arrives at Spider()
-        '''
+        """
 
         if smb_client is None and self.smb_client is None:
-            raise FileRetrievalError('Please specify smb_client')
-
-        #memfile = io.BytesIO()
-        with open(str(self.tmp_filename), 'wb') as f:
+            raise FileRetrievalError("Please specify smb_client")
 
+        # memfile = io.BytesIO()
+        with open(str(self.tmp_filename), "wb") as f:
             try:
                 smb_client.conn.getFile(self.share, self.name, f.write)
             except Exception as e:
@@ -43,9 +41,8 @@ r    '''
                 raise FileRetrievalError(f'Error retrieving file "{str(self)}": {str(e)[:150]}')
 
         # reset cursor back to zero so .read() will return the whole file
-        #memfile.seek(0)
-
+        # memfile.seek(0)
 
     def __str__(self):
 
-        return f'{self.target}\\{self.share}\\{self.name}'
+        return f"{self.target}\\{self.share}\\{self.name}"

man_spider/lib/logger.py

@@ -11,67 +11,62 @@ from logging.handlers import QueueHandler, QueueListener
 
 
 class ColoredFormatter(logging.Formatter):
-
     color_mapping = {
-        'DEBUG':    69, # blue
-        'INFO':     118, # green
-        'WARNING':  208, # orange
-        'ERROR':    196, # red
-        'CRITICAL': 196, # red
+        "DEBUG": 69,  # blue
+        "INFO": 118,  # green
+        "WARNING": 208,  # orange
+        "ERROR": 196,  # red
+        "CRITICAL": 196,  # red
     }
 
     char_mapping = {
-        'DEBUG':    '*',
-        'INFO':     '+',
-        'WARNING':  '-',
-        'ERROR':    '!',
-        'CRITICAL': '!!!',
+        "DEBUG": "*",
+        "INFO": "+",
+        "WARNING": "-",
+        "ERROR": "!",
+        "CRITICAL": "!!!",
     }
 
-    prefix = '\033[1;38;5;'
-    suffix = '\033[0m'
+    prefix = "\033[1;38;5;"
+    suffix = "\033[0m"
 
     def __init__(self, pattern):
 
         super().__init__(pattern)
 
-
     def format(self, record):
 
         colored_record = copy(record)
         levelname = colored_record.levelname
-        levelchar = self.char_mapping.get(levelname, '+')
-        seq = self.color_mapping.get(levelname, 15) # default white
-        colored_levelname = f'{self.prefix}{seq}m[{levelchar}]{self.suffix}'
+        levelchar = self.char_mapping.get(levelname, "+")
+        seq = self.color_mapping.get(levelname, 15)  # default white
+        colored_levelname = f"{self.prefix}{seq}m[{levelchar}]{self.suffix}"
         colored_record.levelname = colored_levelname
 
         return logging.Formatter.format(self, colored_record)
 
-
     @classmethod
     def green(cls, s):
 
         return cls.color(s)
 
-
     @classmethod
     def red(cls, s):
 
-        return cls.color(s, level='ERROR')
-
+        return cls.color(s, level="ERROR")
 
     @classmethod
-    def color(cls, s, level='INFO'):
+    def color(cls, s, level="INFO"):
 
         color = cls.color_mapping.get(level)
-        return f'{cls.prefix}{color}m{s}{cls.suffix}'
-
+        return f"{cls.prefix}{color}m{s}{cls.suffix}"
 
 
 class CustomQueueListener(QueueListener):
-    '''
+    """
     Ignore errors in the monitor thread that result from a race condition when the program exits
-    '''
+    """
+
     def _monitor(self):
         try:
             super()._monitor()
@@ -83,18 +78,18 @@ class CustomQueueListener(QueueListener):
 
 console = logging.StreamHandler(stdout)
 # tell the handler to use this format
-console.setFormatter(ColoredFormatter('%(levelname)s %(message)s'))
+console.setFormatter(ColoredFormatter("%(levelname)s %(message)s"))
 
 ### LOG TO FILE ###
 
 log_queue = Queue()
 listener = CustomQueueListener(log_queue, console)
 sender = QueueHandler(log_queue)
-logging.getLogger('manspider').handlers = [sender]
+logging.getLogger("manspider").handlers = [sender]
 
-logdir = Path.home() / '.manspider' / 'logs'
+logdir = Path.home() / ".manspider" / "logs"
 logdir.mkdir(parents=True, exist_ok=True)
-logfile = f'manspider_{datetime.now().strftime("%m-%d-%Y")}.log'
+logfile = f"manspider_{datetime.now().strftime('%m-%d-%Y')}.log"
 handler = logging.FileHandler(str(logdir / logfile))
-handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
-logging.getLogger('manspider').addHandler(handler)
+handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+logging.getLogger("manspider").addHandler(handler)

man_spider/lib/parser/__init__.py

@@ -1 +1 @@
-from .parser import *
+from .parser import *

man_spider/lib/parser/parser.py

@@ -1,39 +1,80 @@
 import re
-import magic
 import logging
 from time import sleep
 import subprocess as sp
-from extractous import Extractor
+from pathlib import Path
+from kreuzberg import extract_file_sync
+from charset_normalizer import from_path
 
 from man_spider.lib.util import *
 from man_spider.lib.logger import *
 
-log = logging.getLogger('manspider.parser')
+log = logging.getLogger("manspider.parser")
 
 
-class FileParser:
+def is_text_file(filepath):
+    """Detect if file is plain text using charset-normalizer."""
+    result = from_path(filepath)
+    best = result.best()
+    # Only consider it a text file if we have high confidence
+    # and the encoding is detected (not binary)
+    return best is not None and best.encoding is not None
+
+
+def extract_text_file(filepath):
+    """Extract text from plain text file, auto-detecting encoding."""
+    result = from_path(filepath)
+    best = result.best()
+    return str(best) if best else None
+
+
+def extract_strings_from_binary(filepath, min_length=4):
+    """
+    Extract printable ASCII strings from a binary file.
+    Similar to the Unix 'strings' command.
+    """
+    import string
+
+    printable = set(string.printable) - set("\x0b\x0c")  # Exclude vertical tab and form feed
 
-    # don't parse files with these magic types
-    magic_blacklist = [
-        # PNG, JPEG, etc.
-        # 'image data',
-        # ZIP, GZ, etc.
-        'archive data',
-        # encrypted data
-        'encrypted'
-    ]
+    with open(filepath, "rb") as f:
+        data = f.read()
 
+    result = []
+    current = []
+    for byte in data:
+        char = chr(byte) if byte < 128 else None
+        if char and char in printable:
+            current.append(char)
+        else:
+            if len(current) >= min_length:
+                result.append("".join(current))
+            current = []
+    if len(current) >= min_length:
+        result.append("".join(current))
+
+    return "\n".join(result)
+
+
+class FileParser:
+    # don't parse files with these extensions
+    extension_blacklist = {
+        # Archive formats
+        '.zip', '.gz', '.tar', '.bz2', '.7z', '.rar', '.xz', '.tgz', '.tbz2',
+        # Encrypted/protected formats
+        '.enc', '.gpg', '.pgp', '.asc',
+        # Compiled/binary formats that are rarely useful to parse
+        '.exe', '.dll', '.so', '.dylib',
+    }
 
     def __init__(self, filters, quiet=False):
         self.init_content_filters(filters)
-        self.extractor = Extractor()
         self.quiet = quiet
 
-
     def init_content_filters(self, file_content):
-        '''
+        """
         Get ready to search by file content
-        '''
+        """
 
         # strings to look for in file content
         # if empty, content is ignored
@@ -46,41 +87,36 @@ class FileParser:
                 sleep(1)
         if self.content_filters:
             content_filter_str = '"' + '", "'.join([f.pattern for f in self.content_filters]) + '"'
-            log.info(f'Searching by file content: {content_filter_str}')
-
-
+            log.info(f"Searching by file content: {content_filter_str}")
 
     def match(self, file_content):
-        '''
+        """
         Finds all regex matches in file content
-        '''
+        """
 
         for _filter in self.content_filters:
             for match in _filter.finditer(file_content):
                 # ( filter, (match_start_index, match_end_index) )
                 yield (_filter, match.span())
 
-
     def match_magic(self, file):
-        '''
+        """
         Returns True if the file isn't of a blacklisted file type
-        '''
+        """
+        file_path = Path(file)
+        extension = file_path.suffix.lower()
 
-        # get magic type
-        magic_type = magic.from_file(str(file)).lower()
-        for keyword in self.magic_blacklist:
-            if keyword.lower() in magic_type:
-                log.debug(f'Not parsing {file}: blacklisted magic type: "{keyword}"')
-                return False
+        if extension in self.extension_blacklist:
+            log.debug(f'Not parsing {file}: blacklisted extension: "{extension}"')
+            return False
 
         return True
 
-
     def grep(self, content, pattern):
 
         if not self.quiet:
             try:
-                '''
+                """
                 GREP(1)
                     -E, --extended-regexp
                         Interpret PATTERN as an extended regular expression
@@ -90,11 +126,9 @@ class FileParser:
                         Process a binary file as if it were text
                     -m NUM, --max-count=NUM
                         Stop  reading  a file after NUM matching lines
-                '''
+                """
                 grep_process = sp.Popen(
-                    ['grep', '-Eiam', '5', '--color=always', pattern],
-                    stdin=sp.PIPE,
-                    stdout=sp.PIPE
+                    ["grep", "-Eiam", "5", "--color=always", pattern], stdin=sp.PIPE, stdout=sp.PIPE
                 )
                 grep_output = grep_process.communicate(content)[0]
                 for line in grep_output.splitlines():
@@ -102,52 +136,63 @@ class FileParser:
             except (sp.SubprocessError, OSError, IndexError):
                 pass
 
-
     def parse_file(self, file, pretty_filename=None):
-        '''
+        """
         Parse a file on the local filesystem
-        '''
+        """
 
         if pretty_filename is None:
             pretty_filename = str(file)
 
-        log.debug(f'Parsing file: {pretty_filename}')
+        log.debug(f"Parsing file: {pretty_filename}")
 
         matches = dict()
 
         try:
-
-            matches = self.textract(file, pretty_filename=pretty_filename)
+            matches = self.extract_text(file, pretty_filename=pretty_filename)
 
         except Exception as e:
-            #except (BadZipFile, textract.exceptions.CommandLineError) as e:
             if log.level <= logging.DEBUG:
-                log.warning(f'Error extracting text from {pretty_filename}: {e}')
+                log.warning(f"Error extracting text from {pretty_filename}: {e}")
             else:
-                log.warning(f'Error extracting text from {pretty_filename} (-v to debug)')
-            
-        return matches
+                log.warning(f"Error extracting text from {pretty_filename} (-v to debug)")
 
+        return matches
 
-    def textract(self, file, pretty_filename):
-        '''
-        Extracts text from a file
-        Uses the textract library on specific extensions
-        '''
+    def extract_text(self, file, pretty_filename):
+        """
+        Extracts text from a file.
+        Uses charset-normalizer for plain text files (handles UTF-16, etc.)
+        Falls back to kreuzberg for binary formats (docx, pdf, xlsx, etc.)
+        """
 
         matches = dict()
 
-        suffix = Path(str(file)).suffix.lower()
-
         # blacklist certain mime types
         if not self.match_magic(file):
             return matches
 
-        text_content, metadata = self.extractor.extract_file_to_string(str(file))
+        # Try charset-normalizer first for text files (handles UTF-16, etc.)
+        if is_text_file(str(file)):
+            text_content = extract_text_file(str(file))
+            log.debug(f"Extracted text from {pretty_filename} using charset-normalizer")
+        else:
+            # Try kreuzberg for document formats (docx, pdf, xlsx, etc.)
+            try:
+                result = extract_file_sync(str(file))
+                text_content = result.content
+            except Exception as e:
+                # Kreuzberg doesn't support this file type, try extracting raw strings
+                log.debug(f"Kreuzberg failed for {pretty_filename}: {e}, trying string extraction")
+                text_content = extract_strings_from_binary(str(file))
+
+        # Guard against None content
+        if text_content is None:
+            return matches
 
         # try to convert to UTF-8 for grep-friendliness
         try:
-            binary_content = text_content.encode('utf-8', errors='ignore')
+            binary_content = text_content.encode("utf-8", errors="ignore")
         except Exception:
             pass
 

man_spider/lib/processpool.py

@@ -1,28 +1,27 @@
 import logging
-import traceback
 from time import sleep
 import multiprocessing as mp
-from queue import Empty, Full
+from queue import Empty
+from traceback import format_exc
 
 
 # set up logging
-log = logging.getLogger('manspider.processpool')
-
+log = logging.getLogger("manspider.processpool")
 
 
 class ProcessPool:
-    '''
+    """
     usage:
     with ProcessPool(2) as pool:
         for i in pool.map(target, iterable):
             yield i
-    '''
+    """
 
-    def __init__(self, processes=None, daemon=False, name=''):
+    def __init__(self, processes=None, daemon=False, name=""):
 
-        self.name = f'ProcessPool'
+        self.name = "ProcessPool"
         if name:
-            self.name += f'-{name}'
+            self.name += f"-{name}"
 
         if processes is None:
             processes = mp.cpu_count()
@@ -38,16 +37,12 @@ class ProcessPool:
         # make the result queue
         self.result_queue = mp.Manager().Queue()
 
-
     def map(self, func, iterable, args=(), kwargs={}):
 
         # loop until we're out of work
         for entry in iterable:
-
             try:
-
                 while 1:
-
                     for result in self.results:
                         yield result
 
@@ -55,11 +50,15 @@ class ProcessPool:
                     for i in range(len(self.pool)):
                         process = self.pool[i]
                         if process is None or not process.is_alive():
-                            self.pool[i] = mp.Process(target=self.execute, args=(func, self.result_queue, (entry,)+args), \
-                                kwargs=kwargs, daemon=self.daemon)
+                            self.pool[i] = mp.Process(
+                                target=self.execute,
+                                args=(func, self.result_queue, (entry,) + args),
+                                kwargs=kwargs,
+                                daemon=self.daemon,
+                            )
                             self.pool[i].start()
                             self.started_counter += 1
-                            log.debug(f'{self.name}: {self.started_counter:,} processes started')
+                            log.debug(f"{self.name}: {self.started_counter:,} processes started")
                             # success, move on to next
                             assert False
 
@@ -67,26 +66,24 @@ class ProcessPool:
                             yield result
 
                     # prevent unnecessary CPU usage
-                    sleep(.1)
+                    sleep(0.1)
 
             except AssertionError:
                 continue
 
         # wait for processes to finish
         while 1:
-
             finished_threads = [p is None or not p.is_alive() for p in self.pool]
             if all(finished_threads):
                 self.finished_counter += len([p for p in self.pool if p is not None and not p.is_alive()])
                 break
             else:
-                log.debug(f'{self.name}: Waiting for {finished_threads.count(False):,} threads to finish')
+                log.debug(f"{self.name}: Waiting for {finished_threads.count(False):,} threads to finish")
                 sleep(1)
 
         for result in self.results:
             yield result
 
-
     @property
     def results(self):
 
@@ -94,27 +91,25 @@ class ProcessPool:
             try:
                 result = self.result_queue.get_nowait()
                 self.finished_counter += 1
-                log.debug(f'{self.name}: {self.finished_counter:,} processes finished')
+                log.debug(f"{self.name}: {self.finished_counter:,} processes finished")
                 yield result
             except Empty:
-                sleep(.1)
+                sleep(0.1)
                 break
 
-
     @staticmethod
     def execute(func, result_queue, args=(), kwargs={}):
-        '''
+        """
         Executes given function and places return value in result queue
-        '''
+        """
 
         try:
             result_queue.put(func(*args, **kwargs))
         except Exception as e:
             if type(e) not in [FileNotFoundError]:
                 log.critical(format_exc())
-        except KeyboardInterrupt as e:
-            log.critical('ProcessPool Interrupted')
-
+        except KeyboardInterrupt:
+            log.critical("ProcessPool Interrupted")
 
     @staticmethod
     def _close_queue(q):
@@ -126,15 +121,13 @@ class ProcessPool:
                 break
         q.close()
 
-
     def __enter__(self):
 
         return self
 
-
     def __exit__(self, exception_type, exception_value, traceback):
 
         try:
             self._close_queue(self.result_queue)
         except Exception:
-            pass
+            pass