shadowcat 2.0.0__py3-none-any.whl

agent/core/agent.py

@@ -0,0 +1,362 @@
+"""Enhanced Claude Code agent with tracer integration for ShadowCat."""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+# Set up debug logging to file
+# Try to write to workspace first, fallback to /tmp if permission denied
+DEBUG_LOG_WORKSPACE = Path("/workspace/shadowcat_agent-debug.log")
+DEBUG_LOG_FALLBACK = Path("/tmp/shadowcat_agent-debug.log")
+
+handlers: list[logging.Handler] = [logging.StreamHandler()]  # Always log to stderr
+try:
+    # Try to create log file in workspace
+    handlers.append(logging.FileHandler(DEBUG_LOG_WORKSPACE, mode="w"))
+    DEBUG_LOG = DEBUG_LOG_WORKSPACE
+except (PermissionError, OSError):
+    # Fallback to /tmp if workspace is not writable
+    handlers.append(logging.FileHandler(DEBUG_LOG_FALLBACK, mode="w"))
+    DEBUG_LOG = DEBUG_LOG_FALLBACK
+    print(
+        f"Warning: Could not write to {DEBUG_LOG_WORKSPACE}, using {DEBUG_LOG_FALLBACK} instead",
+        flush=True,
+    )
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    handlers=handlers,
+)
+
+# Silence chatty third-party libraries that would otherwise flood the rich UI
+# with DEBUG-level internals (every HTTP request, every selector pick, every
+# JSON payload). Their WARNINGs and ERRORs still come through.
+_NOISY_LOGGERS = (
+    "httpcore",
+    "httpx",
+    "openai",
+    "urllib3",
+    "asyncio",
+    "anyio",
+    "chromadb",
+)
+for _name in _NOISY_LOGGERS:
+    logging.getLogger(_name).setLevel(logging.WARNING)
+
+logger = logging.getLogger(__name__)
+
+
+# Cap on inlined file content. ~30k chars ≈ ~7500 tokens — large enough for
+# most source/config files but bounded so a stray `--file /var/log/syslog`
+# doesn't blow the context budget on turn 1.
+_MAX_CONTEXT_FILE_CHARS = 30_000
+
+# Filenames (or suffixes) that are nearly always autogenerated junk. If --file
+# points at one of these, or a multi-file dump contains a section labelled with
+# one, we strip the section entirely — sending an LLM 500KB of resolved
+# dependency trees is a guaranteed timeout and zero pentest signal.
+_LOCKFILE_NAMES: tuple[str, ...] = (
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "npm-shrinkwrap.json",
+    "cargo.lock",
+    "pipfile.lock",
+    "poetry.lock",
+    "go.sum",
+    "composer.lock",
+    "gemfile.lock",
+    "mix.lock",
+    "podfile.lock",
+)
+_MINIFIED_SUFFIXES: tuple[str, ...] = (".min.js", ".min.css", ".min.mjs", ".map")
+
+# Heuristic markers checked against the first N lines of file content for
+# whole-file autogen rejection. Catches the case where the file is a lockfile
+# but the operator passed it under a different name (e.g. `combined.txt`).
+_AUTOGEN_MARKERS: tuple[str, ...] = (
+    '"lockfileVersion"',  # npm
+    '"@generated"',
+    "DO NOT EDIT",
+    "AUTO-GENERATED",
+    "@generated SignedSource",  # facebook tooling
+    "# This file is automatically @generated by Cargo",
+)
+
+# Per-line filter thresholds
+_MAX_LINE_LEN_NO_WS = 500  # any line longer than this with no whitespace = minified
+_LONG_LINE_SAMPLE = 80  # how many leading chars to inspect for whitespace check
+
+
+def _looks_minified(line: str) -> bool:
+    """A line is 'minified' if it's very long and has no whitespace in the prefix."""
+    if len(line) < _MAX_LINE_LEN_NO_WS:
+        return False
+    return " " not in line[:_LONG_LINE_SAMPLE] and "\t" not in line[:_LONG_LINE_SAMPLE]
+
+
+def _sanitize_context_content(content: str, filename: str = "") -> tuple[str, list[str]]:
+    """Strip junk that wastes tokens (lockfiles, minified code, autogen blobs).
+
+    Two-stage filter:
+      1. Whole-file rejection — if the filename or the file's first few lines
+         look like an autogenerated artefact, return empty content with a
+         single explanatory note. This catches the package-lock.json case
+         that motivated this function.
+      2. Line-level filter — strip individual minified-looking lines (long,
+         whitespace-free), which catches inlined bundles in larger dumps.
+
+    Returns ``(cleaned_content, notes)``. ``notes`` is a list of short
+    human-readable strings describing what was removed; both the operator
+    (via the CLI banner) and the LLM (via an inline annotation) see them.
+    """
+    notes: list[str] = []
+    lower_name = filename.lower()
+
+    # Whole-file rejection by filename suffix
+    for junk in _LOCKFILE_NAMES:
+        if lower_name.endswith(junk):
+            return "", [f"file is a lockfile ({junk}) — content omitted"]
+    for junk in _MINIFIED_SUFFIXES:
+        if lower_name.endswith(junk):
+            return "", [f"file is minified/sourcemap ({junk}) — content omitted"]
+
+    # Whole-file rejection by autogen marker in the first ~4KB
+    head = content[:4096]
+    for marker in _AUTOGEN_MARKERS:
+        if marker in head:
+            return "", [f"file appears autogenerated (marker: {marker!r}) — content omitted"]
+
+    # Line-level filter. keepends=True so we round-trip trailing newlines
+    # exactly — important so the clean-file fast path is byte-identical to
+    # the input and tests can assert that.
+    cleaned_lines: list[str] = []
+    stripped_minified = 0
+    for line in content.splitlines(keepends=True):
+        # Strip trailing newline for the minified check; we want pure
+        # content length, not the platform's line terminator.
+        bare = line.rstrip("\r\n")
+        if _looks_minified(bare):
+            stripped_minified += 1
+            continue
+        cleaned_lines.append(line)
+
+    if stripped_minified:
+        notes.append(f"stripped {stripped_minified} minified/no-whitespace line(s)")
+
+    return "".join(cleaned_lines), notes
+
+
+def build_task(
+    target: str,
+    instruction: str | None = None,
+    hint: str | None = None,
+    context_file: str | None = None,
+) -> str:
+    """Compose the initial user-message task for the agent.
+
+    Layered structure — highest-priority signal goes FIRST so the LLM sees it
+    before the boilerplate task framing:
+
+      1. --hint   → HIGH-PRIORITY OPERATOR HINT block (skip generic recon)
+      2. --file   → PRIMARY CONTEXT FILE block (contents embedded inline)
+      3. standard task framing  (always present)
+      4. --instruction → "Challenge context:" trailer (legacy / lower urgency)
+
+    Args:
+        target:        Target URL / IP / host (always present).
+        instruction:   --instruction value. Background context, lower urgency.
+        hint:          --hint value. Treated as the agent's starting hypothesis.
+        context_file:  Path to a file whose contents become primary evidence.
+                       main.py validates existence before reaching here; this
+                       function degrades gracefully if the path went missing.
+
+    Returns:
+        The composed task string suitable for backend.query().
+    """
+    sections: list[str] = []
+
+    if hint:
+        sections.append(
+            "═══ HIGH-PRIORITY OPERATOR HINT ═══\n"
+            f"{hint}\n"
+            "═══════════════════════════════════\n"
+            "Treat the above as your starting hypothesis. Skip generic "
+            "reconnaissance that does not bear on this hint. Validate the "
+            "hint first; only fall back to broad enumeration if it proves wrong."
+        )
+
+    if context_file:
+        path = Path(context_file)
+        try:
+            original_bytes = path.stat().st_size
+            raw = path.read_text(encoding="utf-8", errors="replace")
+
+            # Stage 1: junk filter. Removes lockfiles, minified code, and
+            # autogen blobs before we even count characters. The motivating
+            # case is `--file combined.txt` where `combined.txt` is a tree
+            # dump that accidentally includes package-lock.json; without
+            # this step we'd happily ship 500KB of resolved deps at the LLM
+            # and trigger the silent OpenRouter timeout this fix exists for.
+            content, notes = _sanitize_context_content(raw, filename=path.name)
+
+            # Stage 2: hard cap as a backstop. If the file is *still* huge
+            # after filtering (e.g. a legitimately enormous log), truncate
+            # with an explicit marker so the LLM doesn't assume it has the
+            # whole picture.
+            truncated_chars = 0
+            if len(content) > _MAX_CONTEXT_FILE_CHARS:
+                truncated_chars = len(content) - _MAX_CONTEXT_FILE_CHARS
+                content = (
+                    content[:_MAX_CONTEXT_FILE_CHARS]
+                    + f"\n\n... [truncated; {truncated_chars:,} more chars not shown]"
+                )
+                notes.append(
+                    f"truncated to first {_MAX_CONTEXT_FILE_CHARS:,} chars "
+                    f"({truncated_chars:,} chars cut)"
+                )
+
+            # Build the section. If the sanitizer rejected the whole file,
+            # tell the LLM clearly rather than embedding an empty code fence.
+            header = (
+                f"═══ PRIMARY CONTEXT FILE: {path.name} ═══\n"
+                f"Operator-provided file. Treat as primary evidence for this challenge.\n"
+                f"Path: {path}  ({original_bytes:,} bytes on disk)"
+            )
+            if notes:
+                header += "\nSanitizer notes: " + "; ".join(notes)
+
+            if content.strip():
+                sections.append(f"{header}\n```\n{content}\n```")
+            else:
+                sections.append(
+                    header + "\n⚠ File was entirely filtered out — ask the operator "
+                    "to pass a more specific file."
+                )
+
+            # Also log what was stripped so it shows up in the debug log /
+            # operator's terminal, not only in the LLM prompt.
+            if notes:
+                logger.warning("context_file %s sanitized: %s", path, "; ".join(notes))
+        except OSError as exc:
+            # main.py validates existence before this is called; this branch
+            # only catches edge cases (file deleted mid-run, permission flip).
+            sections.append(
+                f"⚠ Operator specified --file {context_file} but reading "
+                f"failed: {exc}. Continuing without it."
+            )
+
+    sections.append(f"Solve this CTF challenge and capture the flag(s): {target}")
+
+    if instruction:
+        sections.append(f"Challenge context: {instruction}")
+
+    return "\n\n".join(sections)
+
+
+async def run_pentest(
+    target: str,
+    custom_instruction: str | None = None,
+    model: str | None = None,
+    working_dir: str | None = None,
+    debug: bool = False,
+    resume_session: str | None = None,
+    hint: str | None = None,
+    context_file: str | None = None,
+) -> dict[str, Any]:
+    """
+    Convenience function to run a CTF challenge or penetration test.
+
+    Uses the new AgentController for lifecycle management and session persistence.
+
+    Args:
+        target: Target challenge/machine to solve
+        custom_instruction: Optional custom challenge context or instructions
+        model: Optional model override
+        working_dir: Optional working directory override
+        debug: Enable debug mode with verbose console output
+        resume_session: Optional session ID to resume
+        hint: Optional high-priority hint injected at top of first prompt
+        context_file: Optional path to a file inlined as primary context
+
+    Returns:
+        Dictionary with challenge results including walkthrough and flags found
+    """
+    from agent.core.config import load_config
+    from agent.core.controller import AgentController
+
+    # Enable verbose console logging in debug mode
+    if debug:
+        logger.info("=" * 80)
+        logger.info("DEBUG MODE ENABLED - CTF CHALLENGE SOLVER")
+        logger.info(f"Debug log file: {DEBUG_LOG}")
+        logger.info(f"Target: {target}")
+        if custom_instruction:
+            logger.info(f"Challenge context: {custom_instruction}")
+        if hint:
+            logger.info(f"Operator hint: {hint}")
+        if context_file:
+            logger.info(f"Context file: {context_file}")
+        if model:
+            logger.info(f"Model: {model}")
+        if resume_session:
+            logger.info(f"Resuming session: {resume_session}")
+        logger.info("=" * 80)
+
+    # Build config
+    config_kwargs: dict[str, Any] = {"target": target}
+    if custom_instruction:
+        config_kwargs["custom_instruction"] = custom_instruction
+    if model:
+        config_kwargs["llm_model"] = model
+    if working_dir:
+        config_kwargs["working_directory"] = Path(working_dir)
+
+    config = load_config(**config_kwargs)
+
+    # Build task via shared helper so raw/cli/tui modes stay in sync.
+    task = build_task(
+        target=target,
+        instruction=custom_instruction,
+        hint=hint,
+        context_file=context_file,
+    )
+
+    # Use AgentController for lifecycle management
+    controller = AgentController(config)
+    result = await controller.run(task, resume_session_id=resume_session)
+
+    # Map result to legacy format for backward compatibility
+    if result.get("success"):
+        # Convert flags_found from list of strings to list of dicts if needed
+        flags = result.get("flags_found", [])
+        if flags and isinstance(flags[0], str):
+            flags = [{"flag": f, "context": ""} for f in flags]
+
+        result = {
+            "success": True,
+            "output": result.get("output", ""),
+            "cost_usd": result.get("cost_usd", 0),
+            "flags_found": flags,
+            "walkthrough": [],  # Walkthrough tracking moved to session
+            "session_id": result.get("session_id", ""),
+        }
+
+    if debug:
+        logger.info("=" * 80)
+        logger.info(f"CHALLENGE COMPLETE - Success: {result.get('success')}")
+        if result.get("success"):
+            logger.info(f"Flags found: {len(result.get('flags_found', []))}")
+            logger.info(f"Cost: ${result.get('cost_usd', 0):.4f}")
+            logger.info(f"Session: {result.get('session_id', 'N/A')}")
+            for flag_data in result.get("flags_found", []):
+                flag = (
+                    flag_data.get("flag", flag_data) if isinstance(flag_data, dict) else flag_data
+                )
+                logger.info(f"  🚩 {flag}")
+        else:
+            logger.error(f"Error: {result.get('error', 'Unknown')}")
+        logger.info("=" * 80)
+
+    return result