souleyez 3.0.0__py3-none-any.whl → 3.0.9__py3-none-any.whl

souleyez/engine/background.py

@@ -73,6 +73,10 @@ TRANSIENT_ERROR_PATTERNS = [
     "Resource temporarily unavailable",
     "SMBTimeout",
     "timed out while waiting",
+    # Impacket-specific timeout patterns
+    "] timed out",  # Matches: [Errno Connection error (IP:port)] timed out
+    "RemoteOperations failed",  # Impacket remote operation failures
+    "Errno Connection error",  # Generic Impacket connection errors
 ]
 
 _lock = threading.RLock()  # Reentrant lock allows nested acquisition by same thread
@@ -88,6 +92,40 @@ def _is_transient_error(log_content: str) -> bool:
     return False
 
 
+def _is_netexec_flaky_empty(log_content: str, job: dict) -> bool:
+    """
+    Check if netexec/crackmapexec produced no output (flaky ARM behavior).
+
+    On ARM, netexec is ~20% flaky - it exits 0 but sometimes produces
+    zero output. This detects ONLY that case.
+
+    DOES NOT retry when:
+    - Access denied (has "[-]" error output)
+    - Connected but no shares (legitimate result)
+    - Any netexec output exists
+    """
+    tool = job.get("tool", "").lower()
+    args = job.get("args", [])
+
+    if tool not in ("crackmapexec", "nxc", "netexec"):
+        return False
+
+    args_str = " ".join(args).lower() if args else ""
+    if "--shares" not in args_str and "smb" not in args_str:
+        return False
+
+    lower = log_content.lower()
+
+    # If we got ANY netexec output, it's a real response - don't retry
+    # This includes error output like "[-]" which indicates access denied
+    has_any_output = any(
+        x in lower for x in ["smb", "[*]", "[+]", "[-]", "445", "signing"]
+    )
+
+    # Only retry if truly zero output (ARM flakiness)
+    return not has_any_output
+
+
 class _CrossProcessLock:
     """
     Cross-process file lock using fcntl.flock().
@@ -1516,6 +1554,78 @@ def _is_stdbuf_available() -> bool:
     return _stdbuf_available
 
 
+def _is_python_tool(cmd: List[str]) -> bool:
+    """
+    Check if a command is a Python-based tool.
+
+    stdbuf doesn't work well with Python scripts because Python manages its own
+    buffering internally. On some systems (especially ARM), stdbuf can prevent
+    Python tools from producing any output at all.
+
+    Args:
+        cmd: Command array to check
+
+    Returns:
+        True if the command is a Python tool that should skip stdbuf
+    """
+    if not cmd:
+        return False
+
+    executable = cmd[0]
+
+    # Direct Python invocation
+    if executable in ("python", "python3") or executable.startswith("python"):
+        return True
+
+    # .py extension
+    if executable.endswith(".py"):
+        return True
+
+    # Known Python tools that don't work with stdbuf
+    # These are tools installed via pip/pipx that are Python scripts
+    python_tools = {
+        "netexec",
+        "nxc",
+        "crackmapexec",
+        "cme",
+        "smbmap",
+        "impacket-GetNPUsers",
+        "impacket-GetUserSPNs",
+        "impacket-secretsdump",
+        "impacket-psexec",
+        "impacket-smbclient",
+        "impacket-wmiexec",
+        "impacket-dcomexec",
+        "impacket-atexec",
+        "GetNPUsers.py",
+        "GetUserSPNs.py",
+        "secretsdump.py",
+        "psexec.py",
+        "smbclient.py",
+        "certipy",
+        "bloodhound-python",
+        "ldapdomaindump",
+    }
+
+    # Check base name (handle full paths)
+    base_name = os.path.basename(executable)
+    if base_name in python_tools:
+        return True
+
+    # Check shebang for Python interpreter
+    exe_path = shutil.which(executable)
+    if exe_path:
+        try:
+            with open(exe_path, "rb") as f:
+                first_bytes = f.read(100)
+                if first_bytes.startswith(b"#!") and b"python" in first_bytes:
+                    return True
+        except (IOError, OSError):
+            pass
+
+    return False
+
+
 def _wrap_cmd_for_line_buffering(cmd: List[str]) -> List[str]:
     """
     Wrap a command with stdbuf for line-buffered output when available.
@@ -1524,6 +1634,9 @@ def _wrap_cmd_for_line_buffering(cmd: List[str]) -> List[str]:
     improving real-time log monitoring and ensuring output is captured
     before process termination.
 
+    Note: Python tools are excluded because stdbuf interferes with Python's
+    internal buffering and can cause zero output on some systems (Ubuntu ARM).
+
     Args:
         cmd: Command to wrap
 
@@ -1533,6 +1646,10 @@ def _wrap_cmd_for_line_buffering(cmd: List[str]) -> List[str]:
     if not cmd:
         return cmd
 
+    # Skip stdbuf for Python tools - causes output capture failures on ARM
+    if _is_python_tool(cmd):
+        return cmd
+
     if _is_stdbuf_available():
         # stdbuf -oL = line-buffered stdout, -eL = line-buffered stderr
         return ["stdbuf", "-oL", "-eL"] + cmd
@@ -2064,21 +2181,30 @@ def run_job(jid: int) -> None:
         job = get_job(jid)
         retry_count = job.get("metadata", {}).get("retry_count", 0)
         if retry_count < MAX_RETRIES:
-            # Read log to check for transient errors
-            # Note: Check even when rc==0 because tools like nxc may exit 0 but log errors
+            # Read log to check for transient errors or flaky netexec
             log_path = job.get("log", "")
             if log_path and os.path.exists(log_path):
                 try:
                     with open(log_path, "r", encoding="utf-8", errors="replace") as f:
                         log_content = f.read()
-                    if _is_transient_error(log_content):
-                        # Transient error detected - auto-retry
+
+                    # Check for transient errors OR flaky netexec (connected but no shares)
+                    is_transient = _is_transient_error(log_content)
+                    is_netexec_flaky = _is_netexec_flaky_empty(log_content, job)
+                    retry_reason = None
+
+                    if is_transient:
+                        retry_reason = "transient error"
+                    elif is_netexec_flaky:
+                        retry_reason = "netexec connected but no shares"
+
+                    if retry_reason:
                         logger.info(
-                            "Transient error detected, auto-retrying job",
+                            f"Auto-retrying job: {retry_reason}",
                             extra={"job_id": jid, "retry_count": retry_count + 1},
                         )
                         _append_worker_log(
-                            f"job {jid}: transient error detected, auto-retry {retry_count + 1}/{MAX_RETRIES}"
+                            f"job {jid}: {retry_reason}, auto-retry {retry_count + 1}/{MAX_RETRIES}"
                         )
 
                         # Build new job metadata with incremented retry count
@@ -2095,7 +2221,7 @@ def run_job(jid: int) -> None:
                             engagement_id=job.get("engagement_id"),
                             metadata=new_metadata,
                             parent_id=job.get("metadata", {}).get("parent_id"),
-                            reason=f"Auto-retry {retry_count + 1}/{MAX_RETRIES} (transient error)",
+                            reason=f"Auto-retry {retry_count + 1}/{MAX_RETRIES} ({retry_reason})",
                             rule_id=job.get("metadata", {}).get("rule_id"),
                             skip_scope_check=True,  # Already validated on first run
                         )
@@ -2120,6 +2246,41 @@ def run_job(jid: int) -> None:
 
             # Re-fetch job to get updated data
             job = get_job(jid)
+
+            # Ensure log file is fully flushed to disk before parsing
+            # Some tools (especially Python-based like impacket) may have buffered output
+            log_path = job.get("log", "")
+            if log_path and os.path.exists(log_path):
+                # Wait for completion marker with retries
+                # Log files end with "Exit Code:" or "=== Completed"
+                max_retries = 5
+                retry_delay = 0.2  # 200ms between retries
+                log_complete = False
+
+                for attempt in range(max_retries):
+                    try:
+                        with open(
+                            log_path, "r", encoding="utf-8", errors="replace"
+                        ) as f:
+                            content = f.read()
+                            # Check for completion markers
+                            if "Exit Code:" in content or "=== Completed" in content:
+                                log_complete = True
+                                break
+                    except Exception:
+                        pass
+
+                    if attempt < max_retries - 1:
+                        time.sleep(retry_delay)
+
+                if not log_complete:
+                    # Last resort: force filesystem sync and proceed anyway
+                    try:
+                        with open(log_path, "a") as f:
+                            os.fsync(f.fileno())
+                    except Exception:
+                        pass
+
             parse_result = handle_job_result(job)
 
             # Handle parse failure cases

souleyez/engine/base.py

@@ -3,7 +3,8 @@
 Simple plugin base classes and type hints for souleyez.
 Plugins should implement ScannerPlugin.run(prepared) -> dict (ScanResult).
 """
-from typing import Dict, Any, Optional
+
+from typing import Any, Dict, Optional
 
 ScanResult = Dict[str, Any]
 

souleyez/engine/loader.py

@@ -2,10 +2,12 @@
 """
 Simple plugin loader for souleyez (L1).
 """
+
 from __future__ import annotations
-import pkgutil
+
 import importlib
-from typing import Dict, Any
+import pkgutil
+from typing import Any, Dict
 
 
 def _safe_import_module(fullname: str):

souleyez/engine/log_sanitizer.py

@@ -5,6 +5,7 @@ souleyez.engine.log_sanitizer - Sanitize sensitive data from logs
 Redacts credentials and sensitive information from job logs to prevent
 plaintext password exposure, while maintaining log readability.
 """
+
 import re
 from typing import Optional
 

souleyez/engine/manager.py

@@ -6,11 +6,13 @@ This manager contains a small built-in adapter for "nmap" that uses your existin
 souleyez.scanner.run_nmap function. Later plugins can be added by implementing
 ScannerPlugin in their own modules and registering them here.
 """
+
+import importlib
 import threading
 from typing import Optional
+
 from ..storage.db import init_db, insert_scan, update_scan
 from ..utils import timestamp_str
-import importlib
 
 # lazy import existing run_nmap (your project has souleyez/scanner.py)
 

souleyez/engine/result_handler.py

@@ -2,10 +2,12 @@
 """
 souleyez.engine.result_handler - Auto-parse job results
 """
-import os
+
 import logging
-from typing import Optional, Dict, Any
-from .job_status import STATUS_DONE, STATUS_NO_RESULTS, STATUS_WARNING, STATUS_ERROR
+import os
+from typing import Any, Dict, Optional
+
+from .job_status import STATUS_DONE, STATUS_ERROR, STATUS_NO_RESULTS, STATUS_WARNING
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +51,8 @@ TOOL_ERROR_PATTERNS = {
     "nuclei": [
         "could not connect",
         "context deadline exceeded",
-        "no address found",
+        # Note: "no address found" removed - nuclei reports this as [INF] when
+        # templates try alternate ports, not a real error. Exit code 0 = success.
     ],
     "ffuf": [
         "error making request",
@@ -129,36 +132,11 @@ def handle_job_result(job: Dict[str, Any]) -> Optional[Dict[str, Any]]:
     """
     tool = job.get("tool", "").lower()
     log_path = job.get("log")
-    status = job.get("status")
-
-    # Some tools return non-zero exit codes even on success
-    # Parse 'done' jobs and 'error' jobs for certain tools
-    # gobuster: exits 1 for wildcard detection (but we can still parse and retry)
-    # enum4linux: exits non-zero when some queries fail, even with partial results
-    # msf_exploit: exits 0 even on failure, but we want to parse either way
-    # hashcat: exits 1 when exhausted (no passwords cracked) - not an error
-    # web_login_test: exits 1 when credentials fail - not an error, just invalid creds
-    tools_with_nonzero_success = [
-        "gobuster",
-        "msf_auxiliary",
-        "msfconsole",
-        "enum4linux",
-        "msf_exploit",
-        "wpscan",
-        "hashcat",
-        "web_login_test",
-    ]
-
-    if status == "done":
-        # Always parse successful jobs
-        pass
-    elif status == "error" and tool in tools_with_nonzero_success:
-        # Parse error jobs for tools that can succeed with non-zero exit codes
-        pass
-    else:
-        # Skip other error/failed jobs
-        return None
 
+    # Always try to parse if log file exists
+    # Status checks were causing race conditions where jobs showed "done"
+    # but get_job() returned stale status, causing parse to be skipped
+    # The caller (background.py) only calls us for completed jobs anyway
     if not log_path or not os.path.exists(log_path):
         logger.error(
             f"Job {job.get('id')} parse failed: log file missing or does not exist (path={log_path})"
@@ -193,9 +171,9 @@ def handle_job_result(job: Dict[str, Any]) -> Optional[Dict[str, Any]]:
 
     try:
         from souleyez.handlers.registry import get_handler
-        from souleyez.storage.hosts import HostManager
-        from souleyez.storage.findings import FindingsManager
         from souleyez.storage.credentials import CredentialsManager
+        from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         handler = get_handler(tool)
         if handler:
@@ -306,7 +284,7 @@ def reparse_job(job_id: int) -> Dict[str, Any]:
     Returns:
         Dict with 'success', 'message', and optionally 'parse_result'
     """
-    from .background import get_job, _read_jobs, _write_jobs
+    from .background import _read_jobs, _write_jobs, get_job
 
     # Get the job
     job = get_job(job_id)
@@ -345,9 +323,9 @@ def reparse_job(job_id: int) -> Dict[str, Any]:
 
     try:
         from souleyez.handlers.registry import get_handler
-        from souleyez.storage.hosts import HostManager
-        from souleyez.storage.findings import FindingsManager
         from souleyez.storage.credentials import CredentialsManager
+        from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         handler = get_handler(tool)
         if handler:
@@ -527,10 +505,10 @@ def parse_nmap_job(
 ) -> Dict[str, Any]:
     """Parse nmap job results."""
     try:
+        from souleyez.core.cve_matcher import CVEMatcher
         from souleyez.parsers.nmap_parser import parse_nmap_log
-        from souleyez.storage.hosts import HostManager
         from souleyez.storage.findings import FindingsManager
-        from souleyez.core.cve_matcher import CVEMatcher
+        from souleyez.storage.hosts import HostManager
 
         # Parse the log file
         parsed = parse_nmap_log(log_path)
@@ -804,11 +782,11 @@ def parse_theharvester_job(
     """Parse theHarvester job results."""
     try:
         from souleyez.parsers.theharvester_parser import (
-            parse_theharvester_output,
             get_osint_stats,
+            parse_theharvester_output,
         )
-        from souleyez.storage.osint import OsintManager
         from souleyez.storage.hosts import HostManager
+        from souleyez.storage.osint import OsintManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -1301,14 +1279,15 @@ def parse_gobuster_job(
 ) -> Dict[str, Any]:
     """Parse gobuster job results."""
     try:
+        from urllib.parse import urlparse
+
         from souleyez.parsers.gobuster_parser import (
-            parse_gobuster_output,
-            get_paths_stats,
             generate_next_steps,
+            get_paths_stats,
+            parse_gobuster_output,
         )
-        from souleyez.storage.web_paths import WebPathsManager
         from souleyez.storage.hosts import HostManager
-        from urllib.parse import urlparse
+        from souleyez.storage.web_paths import WebPathsManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -1532,11 +1511,12 @@ def parse_sqlmap_job(
 ) -> Dict[str, Any]:
     """Parse sqlmap job results."""
     try:
-        from souleyez.parsers.sqlmap_parser import parse_sqlmap_output, get_sqli_stats
+        from urllib.parse import urlparse
+
+        from souleyez.parsers.sqlmap_parser import get_sqli_stats, parse_sqlmap_output
         from souleyez.storage.findings import FindingsManager
         from souleyez.storage.hosts import HostManager
         from souleyez.storage.sqlmap_data import SQLMapDataManager
-        from urllib.parse import urlparse
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -1973,9 +1953,9 @@ def _extract_credentials_from_dump(
         tuple: (count of credentials added, list of plaintext credential dicts for chaining)
     """
     from souleyez.intelligence.sensitive_tables import is_sensitive_table
+    from souleyez.log_config import get_logger
     from souleyez.storage.credentials import CredentialsManager
     from souleyez.storage.hosts import HostManager
-    from souleyez.log_config import get_logger
 
     logger = get_logger(__name__)
     cred_manager = CredentialsManager()
@@ -2189,10 +2169,11 @@ def parse_msf_auxiliary_job(
     """Parse MSF auxiliary module job results."""
     try:
         import re
+
         from souleyez.parsers.msf_parser import parse_msf_log
-        from souleyez.storage.hosts import HostManager
-        from souleyez.storage.findings import FindingsManager
         from souleyez.storage.credentials import CredentialsManager
+        from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         # Read raw log for connection failure detection
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -2348,8 +2329,9 @@ def parse_msf_exploit_job(
     """Parse MSF exploit module job results."""
     try:
         import re
-        from souleyez.storage.hosts import HostManager
+
         from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -2501,10 +2483,10 @@ def parse_smbmap_job(
 ) -> Dict[str, Any]:
     """Parse smbmap job results."""
     try:
-        from souleyez.parsers.smbmap_parser import parse_smbmap_output, extract_findings
-        from souleyez.storage.smb_shares import SMBSharesManager
+        from souleyez.parsers.smbmap_parser import extract_findings, parse_smbmap_output
         from souleyez.storage.findings import FindingsManager
         from souleyez.storage.hosts import HostManager
+        from souleyez.storage.smb_shares import SMBSharesManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -2604,9 +2586,9 @@ def parse_whois_job(
     """Parse WHOIS job results."""
     try:
         from souleyez.parsers.whois_parser import (
-            parse_whois_output,
-            map_to_osint_data,
             extract_emails,
+            map_to_osint_data,
+            parse_whois_output,
         )
         from souleyez.storage.osint import OsintManager
 
@@ -2666,8 +2648,8 @@ def parse_dnsrecon_job(
     """Parse dnsrecon job results."""
     try:
         from souleyez.parsers.dnsrecon_parser import parse_dnsrecon_output
-        from souleyez.storage.osint import OsintManager
         from souleyez.storage.hosts import HostManager
+        from souleyez.storage.osint import OsintManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -2818,8 +2800,8 @@ def parse_hydra_job(
     try:
         from souleyez.parsers.hydra_parser import parse_hydra_output
         from souleyez.storage.credentials import CredentialsManager
-        from souleyez.storage.hosts import HostManager
         from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         # Read the log file
         with open(log_path, "r", encoding="utf-8", errors="replace") as f:
@@ -3165,9 +3147,9 @@ def parse_enum4linux_job(
     """Parse enum4linux job results."""
     try:
         from souleyez.parsers.enum4linux_parser import (
-            parse_enum4linux_output,
-            get_smb_stats,
             categorize_share,
+            get_smb_stats,
+            parse_enum4linux_output,
         )
         from souleyez.storage.findings import FindingsManager
         from souleyez.storage.hosts import HostManager
@@ -3310,8 +3292,8 @@ def parse_crackmapexec_job(
     """Parse CrackMapExec job results."""
     try:
         from souleyez.parsers.crackmapexec_parser import parse_crackmapexec
-        from souleyez.storage.hosts import HostManager
         from souleyez.storage.credentials import CredentialsManager
+        from souleyez.storage.hosts import HostManager
 
         target = job.get("target", "")
         parsed = parse_crackmapexec(log_path, target)
@@ -3379,8 +3361,8 @@ def parse_ffuf_job(
     """Parse ffuf job results."""
     try:
         from souleyez.parsers.ffuf_parser import parse_ffuf
-        from souleyez.storage.hosts import HostManager
         from souleyez.storage.findings import FindingsManager
+        from souleyez.storage.hosts import HostManager
 
         target = job.get("target", "")
         parsed = parse_ffuf(log_path, target)
@@ -3709,8 +3691,8 @@ def parse_nikto_job(
     """Parse nikto web server scanner results."""
     try:
         from souleyez.parsers.nikto_parser import (
-            parse_nikto_output,
             generate_next_steps,
+            parse_nikto_output,
         )
         from souleyez.storage.findings import FindingsManager
         from souleyez.storage.hosts import HostManager
@@ -3834,14 +3816,15 @@ def parse_http_fingerprint_job(
     based on detected WAF, CDN, or managed hosting platform.
     """
     try:
+        from urllib.parse import urlparse
+
         from souleyez.parsers.http_fingerprint_parser import (
-            parse_http_fingerprint_output,
             build_fingerprint_context,
-            get_tool_recommendations,
             generate_next_steps,
+            get_tool_recommendations,
+            parse_http_fingerprint_output,
         )
         from souleyez.storage.hosts import HostManager
-        from urllib.parse import urlparse
 
         target = job.get("target", "")
 
@@ -4084,8 +4067,8 @@ def parse_hashcat_job(
     """Parse hashcat job results and extract cracked passwords."""
     try:
         from souleyez.parsers.hashcat_parser import (
-            parse_hashcat_output,
             map_to_credentials,
+            parse_hashcat_output,
         )
         from souleyez.storage.credentials import CredentialsManager
         from souleyez.storage.findings import FindingsManager

souleyez/engine/worker_manager.py

@@ -2,9 +2,11 @@
 """
 Worker health check and management utilities
 """
-import psutil
+
 import time
-from typing import Optional, Tuple, Dict, Any
+from typing import Any, Dict, Optional, Tuple
+
+import psutil
 
 
 def is_worker_running() -> Tuple[bool, Optional[int]]:
@@ -49,7 +51,7 @@ def is_worker_healthy() -> Tuple[bool, Optional[int], Optional[str]]:
         - pid: Worker PID if found, None otherwise
         - issue: Description of issue if not healthy, None otherwise
     """
-    from souleyez.engine.background import get_heartbeat_age, HEARTBEAT_STALE_THRESHOLD
+    from souleyez.engine.background import HEARTBEAT_STALE_THRESHOLD, get_heartbeat_age
 
     is_running, pid = is_worker_running()
 
@@ -169,7 +171,7 @@ def get_worker_health() -> Dict[str, Any]:
         - cpu_percent: CPU usage percentage
         - memory_mb: Memory usage in MB
     """
-    from souleyez.engine.background import get_heartbeat_age, HEARTBEAT_STALE_THRESHOLD
+    from souleyez.engine.background import HEARTBEAT_STALE_THRESHOLD, get_heartbeat_age
 
     is_running, pid = is_worker_running()
     heartbeat_age = get_heartbeat_age()

souleyez/export/evidence_bundle.py

@@ -3,6 +3,7 @@
 Evidence bundle export to ZIP.
 Creates organized ZIP archive of all engagement evidence.
 """
+
 import os
 import zipfile
 from datetime import datetime

souleyez/handlers/base.py

@@ -5,6 +5,7 @@ Base handler class for tool result parsing and display.
 All tool handlers inherit from BaseToolHandler and implement
 the required methods for their specific tool.
 """
+
 import logging
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional

souleyez/handlers/bash_handler.py

@@ -5,6 +5,7 @@ Bash handler.
 Handles bash script execution results, particularly for auto-generated
 scripts like web login credential tests.
 """
+
 import logging
 import os
 import re

souleyez/handlers/bloodhound_handler.py

@@ -3,6 +3,7 @@
 Handler for BloodHound - Active Directory attack path mapping.
 Parses bloodhound-python collection results.
 """
+
 import logging
 import os
 import re

souleyez/handlers/certipy_handler.py

@@ -3,6 +3,7 @@
 Handler for Certipy - Active Directory Certificate Services (ADCS) enumeration.
 Parses vulnerable certificate templates (ESC1-ESC8).
 """
+
 import logging
 import os
 import re