man-spider 1.1.1__tar.gz → 2.0.0__tar.gz

man_spider-2.0.0/.claude/settings.local.json

@@ -0,0 +1,21 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git checkout:*)",
+      "Bash(gh pr checkout:*)",
+      "Bash(git add:*)",
+      "Bash(git commit -m \"$\\(cat <<''EOF''\nMigrate from Poetry to uv package manager\n\n- Change build system from poetry.core to hatchling\n- Convert Poetry-style dependency specifiers to PEP 508 format\n- Move scripts from [tool.poetry.scripts] to [project.scripts]\n- Move repository/homepage to [project.urls]\n- Add uv.lock with updated dependencies\nEOF\n\\)\")",
+      "Bash(git fetch:*)",
+      "Bash(git cherry-pick:*)",
+      "Bash(uv run manspider --help:*)",
+      "Bash(git commit:*)",
+      "Bash(uv sync:*)",
+      "Bash(uv run pytest:*)",
+      "Bash(uv run:*)",
+      "Bash(ls:*)",
+      "Bash(uv add:*)",
+      "Bash(uv run ruff check:*)",
+      "Bash(uv run ruff format:*)"
+    ]
+  }
+}

man_spider-2.0.0/.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  # Monitor Python dependencies (uv)
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "dependencies"
+      - "python"
+
+  # Monitor GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      all-dependencies:
+        patterns:
+          - "*"
+    labels:
+      - "dependencies"
+      - "github-actions"

man_spider-2.0.0/.github/workflows/tests.yml

@@ -0,0 +1,39 @@
+name: Tests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Run ruff check
+        run: uv run ruff check .
+
+      - name: Run ruff format check
+        run: uv run ruff format --check .
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest

man_spider-2.0.0/.kreuzberg/mime/187d2a512b24066d.msgpack

@@ -0,0 +1 @@
+��type�str�data�Gapplication/vnd.openxmlformats-officedocument.wordprocessingml.document�cached_at�A�XY����

man_spider-2.0.0/.kreuzberg/mime/4587374f7aa424a4.msgpack

@@ -0,0 +1 @@
+��type�str�data�image/png�cached_at�A�XY����

man_spider-2.0.0/.kreuzberg/mime/4ea6c5279bb0ec7a.msgpack

@@ -0,0 +1 @@
+��type�str�data�Aapplication/vnd.openxmlformats-officedocument.spreadsheetml.sheet�cached_at�A�XY��rf

man_spider-2.0.0/.kreuzberg/mime/7897c220d66bc1f7.msgpack

@@ -0,0 +1 @@
+��type�str�data�application/pdf�cached_at�A�XY���5

man_spider-2.0.0/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    krb5-user \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-osd \
+    libreoffice \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# Set working directory
+WORKDIR /manspider
+
+# Copy project files
+COPY pyproject.toml uv.lock README.md ./
+COPY man_spider ./man_spider
+
+# Install the package with uv
+RUN uv sync --frozen --no-dev
+
+ENTRYPOINT [".venv/bin/manspider"]

man_spider-1.1.1/README.md → man_spider-2.0.0/PKG-INFO

@@ -1,8 +1,34 @@
+Metadata-Version: 2.4
+Name: man-spider
+Version: 2.0.0
+Summary: Full-featured SMB spider capable of searching file content
+Project-URL: Repository, https://github.com/blacklanternsecurity/MANSPIDER
+Project-URL: Homepage, https://github.com/blacklanternsecurity/MANSPIDER
+Author: TheTechromancer
+License: GPL-3.0
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: charset-normalizer
+Requires-Dist: impacket
+Requires-Dist: kreuzberg
+Description-Content-Type: text/markdown
+
 # MANSPIDER
 ### Crawl SMB shares for juicy information. File content searching + regex is supported!
 
-### UPDATE 2025-05-26
-**Manspider has been updated to support kerberos!** Also, the textract library has been replaced with [Extractous](https://github.com/yobix-ai/extractous), so you can now feel free to run it outside Docker. 🎉
+---
+
+## What's New in v2.0
+
+**Manspider 2.0 is here!** This brings significant improvements:
+
+- **New and improved text extraction** powered by [Kreuzberg](https://github.com/Goldziher/kreuzberg) - now supporting PDF, DOCX, XLSX, PPTX, images with OCR, and many more formats
+- **Modern package management** with [uv](https://github.com/astral-sh/uv) for faster, more reliable installations
+- **Python 3.14 support** - ready for the latest Python releases
+- **Date filtering** - search files by modification date with `--modified-after` and `--modified-before`
+- **Comprehensive unit tests** - improved reliability and stability
+
+---
 
 ![manspider](https://user-images.githubusercontent.com/20261699/74963251-6a08de80-53df-11ea-88f4-60c39665dfa2.gif)
 
@@ -17,19 +43,35 @@
 ### MANSPIDER will crawl every share on every target system. If provided creds don't work, it will fall back to "guest", then to a null session.
 ![manspider](https://user-images.githubusercontent.com/20261699/80316979-f9ab7e80-87ce-11ea-9628-3c22a07e8378.png)
 
-### Installation:
-(Optional) Install these dependencies to add additional file parsing capability:
+### Installation
+
+#### Quick Run with uvx (recommended)
+~~~bash
+# Run directly without installing
+uvx --from git+https://github.com/blacklanternsecurity/MANSPIDER manspider --help
 ~~~
-# for images (png, jpeg)
-$ sudo apt install tesseract-ocr
 
-# for legacy document support (.doc)
-$ sudo apt install antiword
+#### Install with uv
+~~~bash
+# Install globally
+uv tool install git+https://github.com/blacklanternsecurity/MANSPIDER
+
+# Run
+manspider --help
 ~~~
-Install manspider (please be patient, this can take a while):
+
+#### Install with pipx
+~~~bash
+pipx install git+https://github.com/blacklanternsecurity/MANSPIDER
 ~~~
-$ pip install pipx
-$ pipx install git+https://github.com/blacklanternsecurity/MANSPIDER
+
+#### Optional Dependencies
+~~~bash
+# for OCR on images (png, jpeg)
+sudo apt install tesseract-ocr
+
+# for legacy .doc support (kreuzberg uses libreoffice)
+sudo apt install libreoffice
 ~~~
 
 ## Installation (Docker)
@@ -111,6 +153,11 @@ $ manspider share.evilcorp.local -e kdbx kdb 1pif agilekeychain opvault lpd dash
 $ manspider share.evilcorp.local -e pfx p12 pkcs12 pem key crt cer csr jks keystore key keys der -d evilcorp -u bob -p Passw0rd
 ~~~
 
+### Example #10: Search for recently modified files
+~~~
+$ manspider share.evilcorp.local -e docx xlsx pdf --modified-after 2026-01-01 -d evilcorp -u bob -p Passw0rd
+~~~
+
 ### Usage Tip #1:
 You can run multiple instances of manspider at one time. This is useful when one instance is already running, and you want to search what it's downloaded (similar to `grep -R`). To do this, specify the keyword `loot` as the target, which will search the downloaded files in `$HOME/.manspider/loot`.
 
@@ -140,9 +187,9 @@ For example, you could specify any or all of these:
 
 ## Usage:
 ~~~
-usage: manspider [-h] [-u USERNAME] [-p PASSWORD] [-d DOMAIN] [-m MAXDEPTH] [-H HASH] [-t THREADS] [-f REGEX [REGEX ...]] [-e EXT [EXT ...]] [--exclude-extensions EXT [EXT ...]]
-                 [-c REGEX [REGEX ...]] [--sharenames SHARE [SHARE ...]] [--exclude-sharenames [SHARE ...]] [--dirnames DIR [DIR ...]] [--exclude-dirnames DIR [DIR ...]] [-q] [-n]
-                 [-mfail INT] [-o] [-s SIZE] [-v]
+usage: manspider [-h] [-u USERNAME] [-p PASSWORD] [-d DOMAIN] [-l LOOT_DIR] [-m MAXDEPTH] [-H HASH] [-k] [-aesKey HEX] [-dc-ip IP] [-t THREADS] [-f REGEX [REGEX ...]] [-e EXT [EXT ...]]
+                 [--exclude-extensions EXT [EXT ...]] [-c REGEX [REGEX ...]] [--sharenames SHARE [SHARE ...]] [--exclude-sharenames [SHARE ...]] [--dirnames DIR [DIR ...]]
+                 [--exclude-dirnames DIR [DIR ...]] [-q] [-n] [-mfail INT] [-o] [-s SIZE] [-v]
                  targets [targets ...]
 
 Scan for juicy data on SMB shares. Matching files and logs are stored in $HOME/.manspider. All filters are case-insensitive.
@@ -151,26 +198,31 @@ positional arguments:
   targets               IPs, Hostnames, CIDR ranges, or files containing targets to spider (NOTE: local searching also supported, specify directory name or keyword "loot" to search
                         downloaded files)
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
-  -u USERNAME, --username USERNAME
+  -u, --username USERNAME
                         username for authentication
-  -p PASSWORD, --password PASSWORD
+  -p, --password PASSWORD
                         password for authentication
-  -d DOMAIN, --domain DOMAIN
-                        domain for authentication
-  -m MAXDEPTH, --maxdepth MAXDEPTH
+  -d, --domain DOMAIN   domain for authentication
+  -l, --loot-dir LOOT_DIR
+                        loot directory (default ~/.manspider/)
+  -m, --maxdepth MAXDEPTH
                         maximum depth to spider (default: 10)
-  -H HASH, --hash HASH  NTLM hash for authentication
-  -t THREADS, --threads THREADS
+  -H, --hash HASH       NTLM hash for authentication
+  -k, --kerberos        Use Kerberos authentication. Grabs credentials from ccache file (KRB5CCNAME) based on target parameters
+  -aesKey, --aes-key HEX
+                        AES key to use for Kerberos Authentication (128 or 256 bits)
+  -dc-ip, --dc-ip IP    IP Address of the domain controller. If omitted it will use the domain part (FQDN) specified in the target parameter
+  -t, --threads THREADS
                         concurrent threads (default: 5)
-  -f REGEX [REGEX ...], --filenames REGEX [REGEX ...]
+  -f, --filenames REGEX [REGEX ...]
                         filter filenames using regex (space-separated)
-  -e EXT [EXT ...], --extensions EXT [EXT ...]
+  -e, --extensions EXT [EXT ...]
                         only show filenames with these extensions (space-separated, e.g. `docx xlsx` for only word & excel docs)
   --exclude-extensions EXT [EXT ...]
                         ignore files with these extensions
-  -c REGEX [REGEX ...], --content REGEX [REGEX ...]
+  -c, --content REGEX [REGEX ...]
                         search for file content using regex (multiple supported)
   --sharenames SHARE [SHARE ...]
                         only search shares with these names (multiple supported)
@@ -182,10 +234,38 @@ optional arguments:
                         don't search directories containing these strings (multiple supported)
   -q, --quiet           don't display matching file content
   -n, --no-download     don't download matching files
-  -mfail INT, --max-failed-logons INT
+  -mfail, --max-failed-logons INT
                         limit failed logons
   -o, --or-logic        use OR logic instead of AND (files are downloaded if filename OR extension OR content match)
-  -s SIZE, --max-filesize SIZE
+  -s, --max-filesize SIZE
                         don't retrieve files over this size, e.g. "500K" or ".5M" (default: 10M)
   -v, --verbose         show debugging messages
+  --modified-after DATE
+                        only show files modified after this date (format: YYYY-MM-DD)
+  --modified-before DATE
+                        only show files modified before this date (format: YYYY-MM-DD)
 ~~~
+
+## Development
+
+### Testing
+
+```bash
+# Run tests
+uv run pytest
+```
+
+### Linting
+
+This project uses [ruff](https://github.com/astral-sh/ruff) for linting and formatting.
+
+```bash
+# Run linter
+uv run ruff check .
+
+# Run linter with auto-fix
+uv run ruff check --fix .
+
+# Run formatter
+uv run ruff format .
+```

man_spider-2.0.0/PLAN-smb-integration-test.md

@@ -0,0 +1,324 @@
+# MANSPIDER Integration Test Plan
+
+## Overview
+
+Create an end-to-end integration test that:
+1. Spins up an SMB server using impacket's `smbserver.py`
+2. Populates it with test files (various formats and encodings)
+3. Runs MANSPIDER against it
+4. Verifies all files are discovered and text is properly extracted
+
+## Prerequisites
+
+- impacket is already a dependency (used for SMB client in `man_spider/lib/smb.py`)
+- Test data files already exist in `testdata/`
+- pytest is available
+
+## Architecture Understanding
+
+Key files:
+- `man_spider/manspider.py` - CLI entry point, `main()` and `go(options)`
+- `man_spider/lib/spider.py` - MANSPIDER orchestrator class
+- `man_spider/lib/spiderling.py` - Worker that crawls shares and processes files
+- `man_spider/lib/parser/parser.py` - Text extraction via Kreuzberg (line 141: `extract_file_sync()`)
+- `man_spider/lib/smb.py` - Impacket SMB client wrapper
+
+Text extraction flow:
+1. Spiderling downloads file to `/tmp/.manspider/` via `RemoteFile.get()`
+2. `FileParser.extract_text()` calls `extract_file_sync()` from Kreuzberg
+3. Content is matched against regex filters
+
+## Task 1: Clean Up Kreuzberg Tests
+
+Remove text encoding tests from `tests/test_kreuzberg.py` - text handling will be done elsewhere.
+
+**File:** `tests/test_kreuzberg.py`
+
+Keep only:
+```python
+@pytest.mark.parametrize("filename", [
+    "test.docx",
+    "test.pdf",
+    "test.xlsx",
+    "test.png",
+    "test.doc",
+    "test.xls",
+])
+```
+
+Remove: test-ascii.txt, test-utf8.txt, test-utf8-bom.txt, test-utf16le.txt, test-utf16be.txt, test-utf16-bom.txt, test-latin1.txt, test-cp1252.txt
+
+## Task 2: Add Text Encoding Detection to Parser
+
+**File:** `man_spider/lib/parser/parser.py`
+
+Before calling Kreuzberg's `extract_file_sync()`, detect if file is plain text using charset-normalizer.
+
+```python
+from charset_normalizer import from_path
+
+def is_text_file(filepath):
+    """Detect if file is plain text using charset-normalizer."""
+    result = from_path(filepath)
+    return result.best() is not None
+
+def extract_text_file(filepath):
+    """Extract text from plain text file, auto-detecting encoding."""
+    result = from_path(filepath)
+    best = result.best()
+    return str(best) if best else None
+```
+
+Modify `extract_text()` method (around line 130):
+1. First check `is_text_file()`
+2. If True, use `extract_text_file()` directly (handles UTF-16, etc.)
+3. If False, fall back to Kreuzberg for binary formats (docx, pdf, etc.)
+
+**Note:** Check if there's existing encoding detection in `util.py` line 100-109 (`better_decode()`) that uses libmagic - may need to reconcile or replace.
+
+## Task 3: Create SMB Integration Test
+
+**File:** `tests/test_smb_integration.py`
+
+### 3.1 SMB Server Setup
+
+Use impacket's SimpleSMBServer:
+
+```python
+import tempfile
+import shutil
+import threading
+import time
+from pathlib import Path
+from impacket.smbserver import SimpleSMBServer
+
+class SMBTestServer:
+    def __init__(self, share_path, port=4455):
+        self.share_path = share_path
+        self.port = port
+        self.server = None
+        self.thread = None
+
+    def start(self):
+        # Create SMB server with anonymous access
+        self.server = SimpleSMBServer(listenAddress="127.0.0.1", listenPort=self.port)
+        self.server.addShare("testshare", self.share_path, "Test Share")
+
+        # Allow anonymous/guest access
+        self.server.setSMB2Support(True)
+
+        self.thread = threading.Thread(target=self.server.start)
+        self.thread.daemon = True
+        self.thread.start()
+        time.sleep(0.5)  # Wait for server to start
+
+    def stop(self):
+        if self.server:
+            self.server.stop()
+```
+
+### 3.2 Test Fixture
+
+```python
+import pytest
+
+@pytest.fixture
+def smb_server(tmp_path):
+    """Spin up SMB server with test files."""
+    # Copy test files to temp share directory
+    testdata = Path(__file__).parent.parent / "testdata"
+    share_path = tmp_path / "share"
+    share_path.mkdir()
+
+    # Copy all test files
+    for f in testdata.iterdir():
+        if f.is_file():
+            shutil.copy(f, share_path / f.name)
+
+    # Create subdirectory structure for depth testing
+    subdir = share_path / "subdir"
+    subdir.mkdir()
+    shutil.copy(testdata / "test.docx", subdir / "nested.docx")
+
+    server = SMBTestServer(str(share_path), port=4455)
+    server.start()
+    yield server, share_path
+    server.stop()
+```
+
+### 3.3 Integration Tests
+
+```python
+def test_manspider_discovers_all_files(smb_server, tmp_path):
+    """MANSPIDER finds all files on SMB share."""
+    server, share_path = smb_server
+    loot_dir = tmp_path / "loot"
+
+    # Build options similar to CLI
+    from man_spider.lib.spider import MANSPIDER
+
+    # Create options object (see manspider.py for structure)
+    options = create_test_options(
+        targets=["127.0.0.1"],
+        port=4455,
+        loot=str(loot_dir),
+        # Use guest/anonymous auth
+        username="",
+        password="",
+    )
+
+    spider = MANSPIDER(options)
+    spider.start()
+
+    # Verify files were found
+    # Check loot directory or spider output
+    assert (loot_dir / "test.docx").exists()
+    # ... more assertions
+
+
+def test_manspider_extracts_password_from_all_formats(smb_server, tmp_path):
+    """MANSPIDER extracts 'Password123' from various file formats."""
+    server, share_path = smb_server
+    loot_dir = tmp_path / "loot"
+
+    options = create_test_options(
+        targets=["127.0.0.1"],
+        port=4455,
+        loot=str(loot_dir),
+        content=["Password123"],  # Content filter
+    )
+
+    spider = MANSPIDER(options)
+    spider.start()
+
+    # All files containing Password123 should be in loot
+    expected_files = [
+        "test.docx", "test.pdf", "test.xlsx", "test.doc", "test.xls",
+        "test-utf16le.txt", "test-utf8.txt",  # Text files
+    ]
+    for filename in expected_files:
+        assert any(f.name == filename for f in loot_dir.rglob("*")), \
+            f"{filename} not found in loot"
+
+
+def test_manspider_handles_text_encodings(smb_server, tmp_path):
+    """MANSPIDER correctly extracts text from UTF-16 and other encodings."""
+    server, share_path = smb_server
+    loot_dir = tmp_path / "loot"
+
+    options = create_test_options(
+        targets=["127.0.0.1"],
+        port=4455,
+        loot=str(loot_dir),
+        content=["Password123"],
+        extensions=[".txt"],  # Only text files
+    )
+
+    spider = MANSPIDER(options)
+    spider.start()
+
+    # All text encoding variants should match
+    txt_files = list(loot_dir.rglob("*.txt"))
+    assert len(txt_files) >= 8, f"Expected 8 txt files, got {len(txt_files)}"
+```
+
+### 3.4 Helper Function
+
+```python
+def create_test_options(**kwargs):
+    """Create options object matching manspider CLI structure."""
+    from argparse import Namespace
+
+    defaults = {
+        'targets': [],
+        'port': 445,
+        'username': '',
+        'password': '',
+        'domain': '',
+        'hash': '',
+        'loot': '/tmp/manspider_test_loot',
+        'threads': 2,
+        'maxdepth': 10,
+        'content': [],
+        'filenames': [],
+        'extensions': [],
+        'exclude_extensions': [],
+        'dirnames': [],
+        'exclude_dirnames': [],
+        'shares': [],
+        'exclude_shares': [],
+        'max_filesize': '10M',
+        'or_logic': False,
+        'quiet': True,
+        'no_download': False,
+    }
+    defaults.update(kwargs)
+    return Namespace(**defaults)
+```
+
+## Task 4: Test for Binary Files with Embedded Text
+
+Add tests for the binary test files that have "Password123" embedded:
+
+```python
+def test_manspider_finds_password_in_binary(smb_server, tmp_path):
+    """MANSPIDER finds Password123 in arbitrary binary files."""
+    server, share_path = smb_server
+    loot_dir = tmp_path / "loot"
+
+    options = create_test_options(
+        targets=["127.0.0.1"],
+        port=4455,
+        loot=str(loot_dir),
+        content=["Password123"],
+        extensions=[".bin"],
+    )
+
+    spider = MANSPIDER(options)
+    spider.start()
+
+    # Binary files with embedded password should be found
+    bin_files = list(loot_dir.rglob("*.bin"))
+    assert len(bin_files) >= 1
+```
+
+## Implementation Order
+
+1. **Task 1**: Clean up kreuzberg tests (5 min)
+2. **Task 2**: Add charset-normalizer text detection to parser (30 min)
+   - Explore existing `better_decode()` in util.py first
+   - Modify `extract_text()` in parser.py
+   - Run existing tests to verify no regression
+3. **Task 3**: Create SMB integration test infrastructure (1-2 hours)
+   - Set up SMB server fixture
+   - Create helper functions
+   - Write basic discovery test
+   - Write content extraction test
+4. **Task 4**: Add binary file detection tests (15 min)
+
+## Key Files to Modify
+
+| File | Change |
+|------|--------|
+| `tests/test_kreuzberg.py` | Remove text encoding parametrization |
+| `man_spider/lib/parser/parser.py` | Add charset-normalizer detection before Kreuzberg |
+| `tests/test_smb_integration.py` | New file - full integration test suite |
+| `pyproject.toml` | May need to add charset-normalizer if not already dependency |
+
+## Notes
+
+- `charset-normalizer` is already installed (dependency of `requests`)
+- impacket's smbserver requires root/elevated permissions on port 445, so use high port (4455)
+- MANSPIDER uses multiprocessing - tests may need to handle process cleanup
+- Check `man_spider/lib/util.py` line 100-109 for existing `better_decode()` function that uses libmagic for encoding detection - may want to replace with charset-normalizer approach
+
+## Verification
+
+After implementation, run:
+```bash
+uv run pytest tests/ -v
+```
+
+All tests should pass, including:
+- Kreuzberg extraction tests (docx, pdf, xlsx, png, doc, xls)
+- SMB integration tests (file discovery, content extraction, encoding handling)