shadowcat 2.0.0__tar.gz

shadowcat-2.0.0/.env.example

@@ -0,0 +1,35 @@
+# ShadowCat Configuration
+# Copy this file to .env and fill in your values.
+
+# =============================================================================
+# LLM API Key (required)
+#
+# Option A — PSU Blue (university gateway, recommended on campus):
+#   Key format: sk-user-...   — auto-routed to https://ai.psu.blue/v1
+#   Models: qwen/qwen3.6-plus, qwen/qwen3.6-flash, openai/gpt-4o-mini, deepseek/deepseek-chat
+#
+# Option B — OpenRouter (public gateway):
+#   Get your key at https://openrouter.ai/keys   (format: sk-or-...)
+#   Models: any slug from https://openrouter.ai/models
+#
+# Both use the same OPENROUTER_API_KEY variable — the system auto-detects the gateway.
+# =============================================================================
+OPENROUTER_API_KEY=sk-user-your-psu-key-here
+
+# Model slug — must be available on whichever gateway your key targets.
+# PSU Blue examples:   qwen/qwen3.6-plus  |  openai/gpt-4o-mini  |  deepseek/deepseek-chat
+# OpenRouter examples: anthropic/claude-3-5-sonnet  |  openai/gpt-4o
+MODEL=qwen/qwen3.6-plus
+
+# Override the API base URL (optional — auto-detected from key prefix).
+# Set this only if you need to point at a non-standard endpoint.
+# OPENROUTER_BASE_URL=https://ai.psu.blue/v1
+
+# =============================================================================
+# Database (optional) — Enterprise API persistence (app/db)
+# When unset, the API runs in-memory and persists nothing. When set, scans,
+# HTTP evidence, findings, and verdicts are written to PostgreSQL.
+# Standard SQLAlchemy async URL (a plain postgresql:// is upgraded to asyncpg).
+# Percent-encode any @ : / ? characters in the password. Never commit a real one.
+# =============================================================================
+# DATABASE_URL=postgresql+asyncpg://postgres:your-password@127.0.0.1:5432/shadowcat

shadowcat-2.0.0/.gitignore

@@ -0,0 +1,151 @@
+# ============================================================================
+# Python
+# ============================================================================
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# ============================================================================
+# Virtual Environments
+# ============================================================================
+
+.venv/
+venv/
+ENV/
+env/
+env.bak/
+venv.bak/
+
+# ============================================================================
+# Poetry / PDM
+# ============================================================================
+
+poetry.lock
+.pdm.toml
+.pdm-build/
+
+# ============================================================================
+# Testing
+# ============================================================================
+
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.nox/
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+cover/
+
+# ============================================================================
+# Type Checking & Linting
+# ============================================================================
+
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+.ruff_cache/
+cython_debug/
+
+# ============================================================================
+# IDEs & Editors
+# ============================================================================
+
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.aider*
+
+# ============================================================================
+# OS Files
+# ============================================================================
+
+.DS_Store
+Thumbs.db
+
+# ============================================================================
+# Project Specific
+# ============================================================================
+
+# Runtime workspace - NEVER commit (contains VPN configs, exploits, sensitive data)
+workspace/*
+!workspace/.gitkeep
+
+# VPN configuration files (extra safety - never commit these anywhere)
+*.ovpn
+
+# Logs
+*.log
+logs/
+
+# Environment
+.env
+.env.auth
+
+# Agent runs
+agent_runs/
+
+# Legacy project files (when running from legacy/)
+config/chatgpt_config.py
+outputs/
+test_history/
+archive/
+
+# ============================================================================
+# Local Docker Overrides (for regional mirrors, etc.)
+# ============================================================================
+
+Dockerfile.vpn
+docker-compose.override.yml
+vpn-mode.sh
+
+# ============================================================================
+# Documentation
+# ============================================================================
+
+docs/_build/
+/site
+
+# ============================================================================
+# Jupyter
+# ============================================================================
+
+.ipynb_checkpoints
+profile_default/
+ipython_config.py

shadowcat-2.0.0/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "benchmark/xbow-validation-benchmarks"]
+	path = benchmark/xbow-validation-benchmarks
+	url = https://github.com/ThePatrickStar/xbow-validation-benchmarks.git

shadowcat-2.0.0/CLAUDE.md

@@ -0,0 +1,162 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+ShadowCat is an AI-powered autonomous penetration testing agent with a terminal user interface (TUI). It uses an agentic pipeline to solve CTF challenges, Hack The Box machines, and authorized security assessments.
+
+**Stack:** Python 3.12+, uv, Docker (Ubuntu 24.04), Textual (TUI), Rich (CLI), Agent SDK
+
+## Common Commands
+
+```bash
+# Development
+uv sync                           # Install dependencies
+uv run shadowcat-agent --target X # Run the agentic TUI single-shot (CTF/HTB)
+uv run uvicorn backend.api.routes_scan:app --port 8000  # Run the NSC2026 DAST backend
+
+# Local-First Agent OS (daemon + dashboard)
+shadowcat up                      # Start the local daemon and open the dashboard
+shadowcat status                  # Show daemon status (pid, port, uptime)
+shadowcat down                    # Stop the daemon
+
+# Testing
+make test                         # Run all tests
+make test-cov                     # Run tests with coverage
+uv run pytest tests/test_controller.py -v  # Run single test file
+
+# Code Quality
+make lint                         # Run ruff linter
+make format                       # Format code with ruff
+make typecheck                    # Run mypy type checking
+make check                        # All checks (lint + typecheck)
+
+# Docker Workflow
+make install                      # Build Docker image
+make connect                      # Connect to container (main usage)
+make stop                         # Stop container
+make clean-docker                 # Remove everything including config
+```
+
+## Architecture
+
+### Entry Point
+- `agent/interface/main.py` - CLI entry, argument parsing, mode selection (the `shadowcat-agent` command)
+- Command: `shadowcat-agent --target <IP/URL> [--instruction "hint"] [--non-interactive] [--raw] [--debug]`
+- `backend/cli.py` - the `shadowcat` Local-First Agent OS control plane (`up`/`status`/`down`)
+
+### Core Layer (`agent/core/`)
+- **agent.py** - `PentestAgent`: Wraps the LLM agent, handles flag detection, logs to `/workspace/shadowcat-debug.log`
+- **backend.py** - `AgentBackend` interface + `OpenRouterBackend` implementation (OpenAI-compatible)
+- **controller.py** - `AgentController`: 5-state lifecycle (IDLE->RUNNING->PAUSED->COMPLETED->ERROR), pause/resume at message boundaries
+- **events.py** - `EventBus`: Singleton pub/sub for TUI-agent decoupling (STATE_CHANGED, MESSAGE, TOOL, FLAG_FOUND events)
+- **session.py** - `SessionStore`: File-based persistence in `~/.shadowcat/sessions/`, supports session resumption
+- **config.py** - Pydantic settings with `.env` file support
+
+### Interface Layer (`agent/interface/`)
+- **tui.py** - Textual TUI app with real-time activity feed, F1 help, Ctrl+P pause, Ctrl+Q quit
+- **components/** - ActivityFeed, SplashScreen, tool-specific Renderers
+
+### System Prompts (`agent/prompts/`)
+- **pentesting.py** - `CTF_SYSTEM_PROMPT`: CTF methodology, flag formats, persistence directives
+
+## Key Patterns
+
+- **Event-Driven**: TUI subscribes to EventBus; agent emits events for state changes, messages, flags
+- **Singletons**: `EventBus.get()`, `get_global_tracer()` for global access
+- **Abstract Backend**: `AgentBackend` interface allows swapping LLM backends
+- **Flag Detection**: Regex patterns in agent.py match `flag{}`, `HTB{}`, `CTF{}`, 32-char hex
+
+## Testing
+
+Tests use pytest with pytest-asyncio. Mock backends for unit tests.
+
+```bash
+uv run pytest tests/ -v                           # All tests
+uv run pytest tests/test_controller.py -v         # Single file
+uv run pytest tests/test_controller.py::test_name # Single test
+```
+
+## Docker Notes
+
+- Non-root user: `shadowcat` with sudo
+- Workdir: `/workspace` (mounted from `./workspace`)
+- API keys set via `.env.auth` or environment variables
+- Pre-installed: nmap, netcat, curl, wget, git, ripgrep, tmux
+
+## Legacy Version
+
+The previous multi-LLM version (v0.15) is archived in `archive/legacy/`. It supports:
+- OpenAI (GPT-4o, o3, o4-mini)
+- Google Gemini
+- Deepseek
+- Ollama (local LLMs)
+- GPT4All
+
+To develop on the legacy version:
+```bash
+cd archive/legacy
+pip install -e .
+```
+
+## Benchmark System
+
+Use the standalone benchmark runner at `benchmark/standalone-xbow-benchmark-runner/`:
+
+```bash
+cd benchmark/standalone-xbow-benchmark-runner
+
+python3 run_benchmarks.py --range 1-10 --pattern-flag   # Run benchmarks 1-10
+python3 run_benchmarks.py --all --pattern-flag          # Run all 104 benchmarks
+python3 run_benchmarks.py --retry-failed                # Retry failed benchmarks
+python3 run_benchmarks.py --dry-run --range 1-5         # Preview without executing
+```
+
+See `benchmark/standalone-xbow-benchmark-runner/README.md` for full documentation.
+
+## Repository Structure
+
+```
+.
+├── backend/              # NSC2026 entry — enterprise AI DAST (FastAPI + SSE)
+│   ├── api/              # routes_scan.py — scan lifecycle (create/stream/report)
+│   ├── core/             # orchestrator (ReAct loop), llm_client, events
+│   ├── crawler/          # spider, csrf, js_analyzer, subdomains
+│   ├── modes/            # general / enterprise / ctf agent modes + SafetyPolicy
+│   ├── tools/            # http_request, dir_bruteforce, report tools
+│   ├── verification/     # evidence_store + deterministic oracles
+│   ├── waf/              # WAF detection + payload evasion
+│   ├── reporting/        # HTML/PDF report generator
+│   ├── db/               # async SQLAlchemy persistence (optional)
+│   ├── compliance/       # PDPA mapping
+│   └── schemas/          # Pydantic API contracts
+├── agent/               # Agentic TUI package (CTF/HTB) — `shadowcat-agent` CLI; pyproject builds agent + backend
+│   ├── core/             # Agent, controller, events, session
+│   ├── interface/        # TUI and CLI
+│   └── prompts/          # System prompts
+├── docs/                 # ENTERPRISE_ARCH.md, PROGRESS.md, demo/, research/
+├── tests/                # Test suite (backend + agent)
+├── scripts/              # entrypoint.sh, config.sh, test_client.py (manual SSE client)
+├── archive/              # Superseded code (not imported anywhere)
+│   ├── api/              # Earlier DAG/RAG orchestrator attempt
+│   └── legacy/           # Archived v0.15 (multi-LLM)
+├── benchmark/            # Benchmark suites
+│   ├── xbow-validation-benchmarks/  # 104 XBOW benchmarks (git submodule)
+│   └── standalone-xbow-benchmark-runner/  # Benchmark runner
+├── custom_exploits/      # Exploit library (path-pinned by agent prompts)
+├── workspace/            # Runtime workspace (Docker mount)
+├── Dockerfile            # Ubuntu 24.04 container
+├── docker-compose.yml    # Container orchestration
+└── Makefile              # Development commands
+```
+
+> The NSC2026 backend runs with `uv run uvicorn backend.api.routes_scan:app --port 8000`
+> and is consumed by the SecureThai Next.js frontend (separate repo).
+
+## Modification Requirements
+
+When modifying code, ensure:
+- Adherence to existing architecture and patterns
+- Comprehensive tests for new features
+- Ensure to run tests after changes, and do further updates to ensure code quality. Always keep the documentation up to date with any architectural changes. Also ensure all tests pass after modifications.

shadowcat-2.0.0/Dockerfile

@@ -0,0 +1,101 @@
+# ShadowCat Docker Image
+# AI-powered autonomous web security scanner
+
+FROM ubuntu:24.04
+
+LABEL description="ShadowCat - AI-Powered Autonomous Web Security Scanner"
+LABEL version="2.0.0"
+
+# Prevent interactive prompts during build
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update and install system dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+    # Build essentials
+    build-essential \
+    software-properties-common \
+    ca-certificates \
+    gnupg \
+    # Python
+    python3.12 \
+    python3-pip \
+    python3-venv \
+    python3-dev \
+    # Essential security tools
+    nmap \
+    netcat-openbsd \
+    curl \
+    wget \
+    git \
+    sudo \
+    # Network utilities
+    net-tools \
+    dnsutils \
+    whois \
+    # VPN (for authorized connectivity)
+    openvpn \
+    # Text processing
+    jq \
+    ripgrep \
+    # Terminal
+    tmux \
+    && apt-get autoremove -y \
+    && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Remove EXTERNALLY-MANAGED marker to allow pip in Docker
+RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED && \
+    apt-get remove -y python3-cryptography && \
+    apt-get autoremove -y
+
+# Node.js 20 + Claude Code CLI — required by claude-agent-sdk (OAuth auth mode)
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs && \
+    npm install -g @anthropic-ai/claude-code && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install uv for fast Python dependency management
+RUN pip install uv
+
+# Create non-root user
+RUN useradd -m -s /bin/bash shadowcat && \
+    usermod -aG sudo shadowcat && \
+    echo "shadowcat ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Set up working directories
+RUN mkdir -p /workspace /app /home/shadowcat/.claude && \
+    chown -R shadowcat:shadowcat /workspace /app /home/shadowcat/.claude && \
+    chmod 700 /home/shadowcat/.claude
+
+# Switch to shadowcat user
+USER shadowcat
+WORKDIR /app
+
+# Copy project files
+COPY --chown=shadowcat:shadowcat pyproject.toml uv.lock README.md /app/
+COPY --chown=shadowcat:shadowcat agent/ /app/agent/
+COPY --chown=shadowcat:shadowcat backend/ /app/backend/
+COPY --chown=shadowcat:shadowcat scripts/entrypoint.sh /home/shadowcat/entrypoint.sh
+
+# Install Python dependencies via uv (frozen = reproducible)
+USER root
+RUN uv sync --frozen --no-dev && \
+    chmod +x /home/shadowcat/entrypoint.sh
+
+# Switch back to shadowcat user for runtime
+USER shadowcat
+
+# Set environment variables
+ENV PYTHONPATH=/app
+ENV PYTHONUNBUFFERED=1
+
+# Default working directory
+WORKDIR /workspace
+
+# Use entrypoint script for auth setup
+ENTRYPOINT ["/home/shadowcat/entrypoint.sh"]
+
+# Default command - interactive bash
+CMD ["/bin/bash"]

shadowcat-2.0.0/LEARNING.md

@@ -0,0 +1,61 @@
+# LEARNING.md — แผนเรียนรู้ ShadowCat เพื่อ NSC ครั้งที่ 28
+
+> ไฟล์นี้คือ "แผนการเรียน" ของเจ้าของโปรเจค เป้าหมายคือ **อ่านโค้ดตัวเองออก
+> และอธิบาย/ป้องกันงานบนเวทีแข่งได้** — ไม่ใช่แค่ส่งงานที่ AI ทำให้
+
+---
+
+## เป้าหมาย (ปลายทาง)
+1. อ่านโค้ดในโปรเจคนี้แล้วเข้าใจว่ามันทำอะไร ทำไมถึงเขียนแบบนั้น
+2. ตอบกรรมการได้เวลาถูกถาม "ตรงนี้ทำงานยังไง / ทำไมทำแบบนี้"
+3. แก้/ต่อยอดโค้ดเองได้บ้าง ไม่ใช่ต้องพึ่ง AI 100%
+
+## กฎการทำงานของเรา (โหมดสอน)
+- **AI สอน เจ้าของลงมือ** — จะไม่เขียนให้เฉยๆ อีกแล้ว
+- เรียนผ่าน **โค้ดจริงในโปรเจคนี้** (`backend/`) ไม่ใช่คอร์ส Python ลอยๆ
+- ทุกบทจบด้วย **"อธิบายกลับให้ฟัง"** — อธิบายได้ = เข้าใจจริง = ตอบกรรมการได้
+- งงตรงไหน **พูดเลย ไม่ต้องเดาให้สวย** — การบอกว่า "ตรงนี้ไม่เข้าใจ" คือก้าวที่เร็วที่สุด
+
+## คุณอยู่ตรงไหนตอนนี้ (เช็คเมื่อ 2026-06-13)
+- ✅ อ่านค่าตรงๆ ในโค้ดได้ (`return`, อ่าน key/value ใน dict ถูก)
+- ✅ เริ่มจับทักษะ **"แยกบรรทัดเป็นชิ้นๆ"** และโยง type hint เข้ากับค่าจริงได้
+- ⏳ ยังต้องสร้างคำศัพท์พื้นฐาน: `def`/function, decorator (`@`), `async`, type hint, `dict`/`list`
+
+---
+
+## เส้นทาง (ทำทีละโมดูลตามลำดับ)
+รูปแบบแต่ละโมดูล:  **[อ่านไฟล์ไหน] → [ต้องเข้าใจอะไร] → [ข้อสอบ: อธิบายกลับ]**
+
+- [x] **โมดูล 0 — อ่าน 1 บรรทัดเป็นชิ้นๆ + dict คือ key→value**  *(ผ่านแล้ว 2026-06-13)*
+- [x] **โมดูล 1 — ตัวแปร & ชนิดข้อมูล**: `str` `int` `bool` `dict` `list` + อ่าน `dict[K,V]`/`list[T]`  *(ผ่านแล้ว 2026-06-14)*
+- [ ] **โมดูล 2 — ฟังก์ชัน**: `def` คืออะไร, รับ input (arguments), `return` ค่ากลับ
+- [ ] **โมดูล 3 — เว็บเบื้องต้น (FastAPI)**: route, decorator `@app.get` ทำงานยังไง
+      → ไฟล์ `backend/api/routes_scan.py` (เริ่มจาก `/health`)
+- [ ] **โมดูล 4 — `async`/`await`**: ทำไมต้องมี (สแกนยิง HTTP หลายอันพร้อมกัน)
+      → ไฟล์ `backend/tools/http_tool.py`
+- [ ] **โมดูล 5 — หัวใจของโปรเจค: agent ReAct loop** (คิด → เรียกเครื่องมือ → ดูผล → วนซ้ำ)
+      → ไฟล์ `backend/core/orchestrator.py`  ← ตัวที่กรรมการน่าจะถามมากสุด
+- [ ] **โมดูล 6 — สัญญาข้อมูล (Pydantic schemas)**: รูปร่างของ request/response
+      → ไฟล์ `backend/schemas/api.py`
+- [ ] **โมดูล 7 — Live Feed ทำงานยังไง (SSE events)**
+      → ไฟล์ `backend/core/events.py`
+- [ ] **โมดูล 8 — ของที่เราเพิ่งสร้าง**: scan_memory (ความจำข้ามสแกน), coverage, oracle (ตัวตัดสิน finding)
+
+---
+
+## ข้อสอบเวที — คำถามที่กรรมการน่าจะถาม (เป้าหมายว่าต้องตอบได้)
+ฝึกตอบทีละข้อเมื่อเรียนถึงโมดูลที่เกี่ยว:
+1. โปรเจคนี้ทำอะไร? อธิบายใน 30 วินาที
+2. agent มัน "คิด" และ "ลงมือ" ยังไง? (ReAct loop — โมดูล 5)
+3. กันไม่ให้มันสแกนนอกขอบเขตที่ได้รับอนุญาตยังไง? (SafetyPolicy)
+4. ทำไม finding ถึงเชื่อถือได้? (oracle เป็นคนตัดสิน ไม่ใช่ AI เดาเอง — โมดูล 8)
+5. ประหยัด cost / token ยังไง? (pruning, scan memory)
+6. "ความจำข้ามสแกน" ทำงานยังไง ทำไมยิ่งสแกนยิ่งฉลาด? (โมดูล 8)
+7. ใช้ AI สร้างเยอะ แล้วส่วนไหนที่คุณเข้าใจ/ออกแบบเอง? ← ตอบตรงๆ ได้คือแต้มต่อ
+
+---
+
+## บันทึกความคืบหน้า (อัปเดตทุกครั้งที่เรียน)
+- **2026-06-13** — เริ่มโหมดสอน. ผ่านโมดูล 0 (แยกบรรทัด + dict key/value).
+- **2026-06-14** — ผ่านโมดูล 1: ชนิดข้อมูล 5 ตัว (`str` `int` `bool` `list` `dict`) + อ่าน
+  type hint `dict[str,str]` และ `list[str]` ได้เองโดยไม่ต้องใบ้. ต่อไป: โมดูล 2 (ฟังก์ชัน).

shadowcat-2.0.0/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ShadowCat Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.