shadowcat 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. agent/__init__.py +17 -0
  2. agent/benchmark/__init__.py +11 -0
  3. agent/benchmark/cli.py +179 -0
  4. agent/benchmark/config.py +15 -0
  5. agent/benchmark/docker.py +192 -0
  6. agent/benchmark/registry.py +99 -0
  7. agent/core/__init__.py +0 -0
  8. agent/core/agent.py +362 -0
  9. agent/core/backend.py +1667 -0
  10. agent/core/config.py +106 -0
  11. agent/core/controller.py +638 -0
  12. agent/core/events.py +177 -0
  13. agent/core/langfuse.py +320 -0
  14. agent/core/phantom.py +2327 -0
  15. agent/core/planner.py +493 -0
  16. agent/core/profiling.py +58 -0
  17. agent/core/sanitizer.py +104 -0
  18. agent/core/session.py +228 -0
  19. agent/core/tracer.py +137 -0
  20. agent/interface/__init__.py +0 -0
  21. agent/interface/components/__init__.py +0 -0
  22. agent/interface/components/activity_feed.py +202 -0
  23. agent/interface/components/renderers.py +149 -0
  24. agent/interface/components/splash.py +112 -0
  25. agent/interface/main.py +1126 -0
  26. agent/interface/styles.tcss +421 -0
  27. agent/interface/tui.py +508 -0
  28. agent/parsing/html_distiller.py +230 -0
  29. agent/parsing/tool_parser.py +115 -0
  30. agent/prompts/__init__.py +0 -0
  31. agent/prompts/pentesting.py +238 -0
  32. agent/rag_module/knowledge_base.py +116 -0
  33. agent/tests/benchmark_phantom.py +455 -0
  34. agent/tools/__init__.py +14 -0
  35. agent/tools/base.py +99 -0
  36. agent/tools/executor.py +345 -0
  37. agent/tools/registry.py +47 -0
  38. backend/README.md +244 -0
  39. backend/__init__.py +10 -0
  40. backend/api/__init__.py +1 -0
  41. backend/api/routes_scan.py +932 -0
  42. backend/authz.py +75 -0
  43. backend/cli.py +261 -0
  44. backend/compliance/__init__.py +1 -0
  45. backend/compliance/pdpa_mapping.py +16 -0
  46. backend/core/__init__.py +1 -0
  47. backend/core/coverage.py +93 -0
  48. backend/core/events.py +166 -0
  49. backend/core/llm_client.py +614 -0
  50. backend/core/orchestrator.py +402 -0
  51. backend/core/scan_memory.py +236 -0
  52. backend/crawler/__init__.py +1 -0
  53. backend/crawler/csrf.py +75 -0
  54. backend/crawler/headless.py +232 -0
  55. backend/crawler/js_analyzer.py +133 -0
  56. backend/crawler/spider.py +550 -0
  57. backend/crawler/subdomains.py +279 -0
  58. backend/daemon.py +252 -0
  59. backend/db/README.md +86 -0
  60. backend/db/__init__.py +17 -0
  61. backend/db/engine.py +87 -0
  62. backend/db/models.py +206 -0
  63. backend/db/repository.py +316 -0
  64. backend/db/schema.sql +247 -0
  65. backend/modes/__init__.py +5 -0
  66. backend/modes/base.py +178 -0
  67. backend/modes/ctf.py +39 -0
  68. backend/modes/enterprise.py +210 -0
  69. backend/modes/general.py +226 -0
  70. backend/modes/registry.py +34 -0
  71. backend/reporting/__init__.py +0 -0
  72. backend/reporting/generator.py +285 -0
  73. backend/schemas/__init__.py +1 -0
  74. backend/schemas/api.py +423 -0
  75. backend/tools/__init__.py +62 -0
  76. backend/tools/dirbrute_tool.py +304 -0
  77. backend/tools/general_report_tool.py +135 -0
  78. backend/tools/http_tool.py +351 -0
  79. backend/tools/nuclei_tool.py +20 -0
  80. backend/tools/report_tool.py +145 -0
  81. backend/tools/shell_tools.py +23 -0
  82. backend/verification/__init__.py +1 -0
  83. backend/verification/evidence_store.py +125 -0
  84. backend/verification/general_oracle.py +369 -0
  85. backend/verification/idor_oracle.py +131 -0
  86. backend/waf/__init__.py +0 -0
  87. backend/waf/detector.py +147 -0
  88. backend/waf/evasion.py +117 -0
  89. backend/webui/index.html +713 -0
  90. backend/workspace.py +182 -0
  91. shadowcat-2.0.0.dist-info/METADATA +360 -0
  92. shadowcat-2.0.0.dist-info/RECORD +95 -0
  93. shadowcat-2.0.0.dist-info/WHEEL +4 -0
  94. shadowcat-2.0.0.dist-info/entry_points.txt +4 -0
  95. shadowcat-2.0.0.dist-info/licenses/LICENSE.md +21 -0
agent/core/backend.py ADDED
@@ -0,0 +1,1667 @@
1
+ """Framework-agnostic agent backend protocol and implementations."""
2
+
3
+ import asyncio
4
+ import json
5
+ import os
6
+ import shlex
7
+ import time
8
+ from abc import ABC, abstractmethod
9
+ from collections.abc import AsyncIterator
10
+ from dataclasses import dataclass, field
11
+ from enum import Enum
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from agent.core.profiling import mark, span
16
+ from agent.core.sanitizer import scrubbed_env, wrap_untrusted
17
+
18
+
19
+ class MessageType(Enum):
20
+ """Framework-agnostic message types from agent backends."""
21
+
22
+ TEXT = "text"
23
+ TOOL_START = "tool_start"
24
+ TOOL_RESULT = "tool_result"
25
+ RESULT = "result"
26
+ ERROR = "error"
27
+
28
+
29
+ @dataclass
30
+ class AgentMessage:
31
+ """Framework-agnostic message from any agent backend."""
32
+
33
+ type: MessageType
34
+ content: Any
35
+ tool_name: str | None = None
36
+ tool_args: dict[str, Any] | None = None
37
+ metadata: dict[str, Any] = field(default_factory=dict)
38
+
39
+
40
+ class AgentBackend(ABC):
41
+ """
42
+ Abstract interface for agent backends.
43
+
44
+ Implement this to support different LLM providers:
45
+ - OpenRouterBackend (current — OpenAI-compatible Chat Completions via OpenRouter)
46
+ - Any future provider that speaks the OpenAI protocol
47
+ """
48
+
49
+ @abstractmethod
50
+ async def connect(self) -> None:
51
+ """Establish connection to the agent."""
52
+ ...
53
+
54
+ @abstractmethod
55
+ async def disconnect(self) -> None:
56
+ """Close connection."""
57
+ ...
58
+
59
+ @abstractmethod
60
+ async def query(self, prompt: str) -> None:
61
+ """Send a query/instruction to the agent."""
62
+ ...
63
+
64
+ @abstractmethod
65
+ def receive_messages(self) -> AsyncIterator[AgentMessage]:
66
+ """Async iterator yielding messages from agent."""
67
+ ...
68
+
69
+ @property
70
+ @abstractmethod
71
+ def session_id(self) -> str | None:
72
+ """Current session ID (if backend supports sessions)."""
73
+ ...
74
+
75
+ @property
76
+ def supports_resume(self) -> bool:
77
+ """Whether this backend supports session resume."""
78
+ return False
79
+
80
+ @abstractmethod
81
+ async def resume(self, session_id: str) -> bool:
82
+ """Resume a previous session. Returns success."""
83
+ ...
84
+
85
+
86
+ # ---------------------------------------------------------------------------
87
+ # Tool schemas (OpenAI function-calling format)
88
+ # ---------------------------------------------------------------------------
89
+
90
+ _TOOLS: list[dict[str, Any]] = [
91
+ {
92
+ "type": "function",
93
+ "function": {
94
+ "name": "bash",
95
+ "description": (
96
+ "Execute a bash command in the working directory. "
97
+ "Use this for network enumeration, exploitation, running scripts, "
98
+ "reading output of security tools (nmap, curl, python, etc.)."
99
+ ),
100
+ "parameters": {
101
+ "type": "object",
102
+ "properties": {
103
+ "command": {
104
+ "type": "string",
105
+ "description": "The bash command to execute.",
106
+ }
107
+ },
108
+ "required": ["command"],
109
+ },
110
+ },
111
+ },
112
+ {
113
+ "type": "function",
114
+ "function": {
115
+ "name": "read_file",
116
+ "description": "Read the contents of a file.",
117
+ "parameters": {
118
+ "type": "object",
119
+ "properties": {
120
+ "path": {
121
+ "type": "string",
122
+ "description": "Path to the file (absolute, or relative to working directory).",
123
+ }
124
+ },
125
+ "required": ["path"],
126
+ },
127
+ },
128
+ },
129
+ {
130
+ "type": "function",
131
+ "function": {
132
+ "name": "write_file",
133
+ "description": "Write content to a file, creating parent directories as needed.",
134
+ "parameters": {
135
+ "type": "object",
136
+ "properties": {
137
+ "path": {
138
+ "type": "string",
139
+ "description": "Destination path.",
140
+ },
141
+ "content": {
142
+ "type": "string",
143
+ "description": "Text content to write.",
144
+ },
145
+ },
146
+ "required": ["path", "content"],
147
+ },
148
+ },
149
+ },
150
+ {
151
+ "type": "function",
152
+ "function": {
153
+ "name": "http_request",
154
+ "description": (
155
+ "Send an HTTP request through a persistent session. Cookies, "
156
+ "auth headers, and CSRF tokens carry over between calls "
157
+ "automatically — use this instead of curl/wget for any web "
158
+ "request. HTML responses are distilled (scripts/styles/SVG "
159
+ "stripped; forms, links, comments, meta, headings preserved) "
160
+ "to keep token usage low. Returns structured JSON with status, "
161
+ "filtered headers, distilled body, cookies set, elapsed_ms, "
162
+ "and the redirect chain."
163
+ ),
164
+ "parameters": {
165
+ "type": "object",
166
+ "properties": {
167
+ "method": {
168
+ "type": "string",
169
+ "enum": ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"],
170
+ },
171
+ "url": {"type": "string", "description": "Absolute URL."},
172
+ "headers": {
173
+ "type": "object",
174
+ "additionalProperties": {"type": "string"},
175
+ "description": "Extra request headers. Persistent session "
176
+ "headers (cookies, prior Authorization) are sent automatically.",
177
+ },
178
+ "body": {
179
+ "type": "string",
180
+ "description": "Raw request body. Mutually exclusive with form/json_body.",
181
+ },
182
+ "form": {
183
+ "type": "object",
184
+ "description": "Form-encoded fields. Sets Content-Type to "
185
+ "application/x-www-form-urlencoded.",
186
+ },
187
+ "json_body": {
188
+ "type": "object",
189
+ "description": "JSON body. Sets Content-Type to application/json.",
190
+ },
191
+ "follow_redirects": {"type": "boolean", "default": True},
192
+ "auto_csrf": {
193
+ "type": "boolean",
194
+ "default": True,
195
+ "description": "Inject a CSRF token previously seen for this "
196
+ "form action if the request doesn't already carry one.",
197
+ },
198
+ "timeout_seconds": {"type": "number", "default": 30},
199
+ },
200
+ "required": ["method", "url"],
201
+ },
202
+ },
203
+ },
204
+ {
205
+ "type": "function",
206
+ "function": {
207
+ "name": "hash_crack",
208
+ "description": "Crack a hash using hashcat/rockyou under the hood.",
209
+ "parameters": {
210
+ "type": "object",
211
+ "properties": {
212
+ "hash_value": {
213
+ "type": "string",
214
+ "description": "The hash string to crack.",
215
+ }
216
+ },
217
+ "required": ["hash_value"],
218
+ },
219
+ },
220
+ },
221
+ {
222
+ "type": "function",
223
+ "function": {
224
+ "name": "decode_chain",
225
+ "description": "Recursively decode base64/hex/url encoding.",
226
+ "parameters": {
227
+ "type": "object",
228
+ "properties": {
229
+ "encoded_string": {
230
+ "type": "string",
231
+ "description": "The encoded string to decode.",
232
+ }
233
+ },
234
+ "required": ["encoded_string"],
235
+ },
236
+ },
237
+ },
238
+ {
239
+ "type": "function",
240
+ "function": {
241
+ "name": "sqli_probe",
242
+ "description": "Probe a URL for SQL injection using sqlmap.",
243
+ "parameters": {
244
+ "type": "object",
245
+ "properties": {
246
+ "url": {
247
+ "type": "string",
248
+ "description": "The target URL to probe.",
249
+ }
250
+ },
251
+ "required": ["url"],
252
+ },
253
+ },
254
+ },
255
+ ]
256
+
257
+ _TOOL_TIMEOUT_SECONDS = 120
258
+ _OUTPUT_TRUNCATE_CHARS = 12_000
259
+
260
+ # Hard ceiling on tool-call turns within one `receive_messages` call. Without
261
+ # this, a model stuck in a degenerate "call → fail → call again" loop can
262
+ # burn budget indefinitely under the LLM-timeout backstop.
263
+ _MAX_AGENT_TURNS = 60
264
+
265
+ # Hard ceiling on a single LLM round-trip (model call). Applied at two layers:
266
+ # 1. The SDK's own httpx timeout (`AsyncOpenAI(timeout=...)`)
267
+ # 2. An `asyncio.wait_for` wrapping the create() call.
268
+ # Two layers because (1) catches socket-level stalls but can be defeated by a
269
+ # server that streams keepalive bytes forever; (2) is the unconditional ceiling.
270
+ # The wait_for budget is slightly larger so the SDK's own timeout fires first
271
+ # and surfaces a clean httpx.TimeoutException rather than a CancelledError.
272
+ _LLM_REQUEST_TIMEOUT_SECONDS = 180
273
+ _LLM_WAITFOR_TIMEOUT_SECONDS = 210
274
+
275
+
276
+ # ---------------------------------------------------------------------------
277
+ # Prompt caching helpers
278
+ # ---------------------------------------------------------------------------
279
+ #
280
+ # The system prompt (the large CTF methodology) and the tool schemas are
281
+ # byte-identical on every one of the up-to-_MAX_AGENT_TURNS calls in a session.
282
+ # Flagging that stable prefix with Anthropic's `cache_control: ephemeral` turns
283
+ # it into a cache read (~10% of input price) after the first turn — the single
284
+ # biggest cost lever for a multi-turn agent. Native Claude supports this
285
+ # directly; OpenRouter forwards the field to Anthropic/Gemini upstreams. Other
286
+ # providers are left untouched (they auto-cache a byte-stable prefix anyway).
287
+
288
+ _CACHE_CONTROL_EPHEMERAL: dict[str, str] = {"type": "ephemeral"}
289
+
290
+
291
+ def _is_anthropic_model(model: str) -> bool:
292
+ """True for Claude models (native names or OpenRouter's `anthropic/` slug)."""
293
+ m = (model or "").lower()
294
+ return "claude" in m or "anthropic" in m
295
+
296
+
297
+ def _cacheable_system_message(system_prompt: str) -> dict[str, Any]:
298
+ """OpenAI-format system message with its text part flagged for caching."""
299
+ return {
300
+ "role": "system",
301
+ "content": [
302
+ {
303
+ "type": "text",
304
+ "text": system_prompt,
305
+ "cache_control": _CACHE_CONTROL_EPHEMERAL,
306
+ }
307
+ ],
308
+ }
309
+
310
+
311
+ def _build_cached_tools(tools: list[dict[str, Any]], model: str) -> list[dict[str, Any]]:
312
+ """Copy ``tools`` with a cache breakpoint on the last entry for Claude models.
313
+
314
+ Never mutates the shared ``_TOOLS`` global. Non-Anthropic models get the
315
+ list back unchanged (they reject or ignore the field).
316
+ """
317
+ if not tools or not _is_anthropic_model(model):
318
+ return tools
319
+ cached = list(tools)
320
+ cached[-1] = {**cached[-1], "cache_control": _CACHE_CONTROL_EPHEMERAL}
321
+ return cached
322
+
323
+
324
+ _GLOBAL_HTTP_CLIENT = None
325
+ _GLOBAL_CSRF_JAR: dict[str, str] = {}
326
+
327
+
328
+ def get_shared_http_client() -> Any:
329
+ """Get or create the shared HTTP client for the event loop."""
330
+ global _GLOBAL_HTTP_CLIENT
331
+ if _GLOBAL_HTTP_CLIENT is not None:
332
+ return _GLOBAL_HTTP_CLIENT
333
+
334
+ try:
335
+ import httpx
336
+ except ImportError as exc:
337
+ raise RuntimeError("The 'httpx' package is required. Install with: uv add httpx") from exc
338
+
339
+ proxy = os.getenv("SHADOWCAT_PROXY")
340
+ http_kwargs: dict[str, Any] = {
341
+ "timeout": httpx.Timeout(30.0, connect=10.0),
342
+ "follow_redirects": True,
343
+ "headers": {"User-Agent": "ShadowCat/1.0 (+autonomous-pentest)"},
344
+ "verify": False,
345
+ }
346
+ if proxy:
347
+ http_kwargs["proxy"] = proxy
348
+ _GLOBAL_HTTP_CLIENT = httpx.AsyncClient(**http_kwargs)
349
+ return _GLOBAL_HTTP_CLIENT
350
+
351
+
352
+ def get_shared_csrf_jar() -> dict[str, str]:
353
+ return _GLOBAL_CSRF_JAR
354
+
355
+
356
+ class OpenRouterBackend(AgentBackend):
357
+ """
358
+ OpenRouter backend using the OpenAI-compatible Chat Completions API.
359
+
360
+ Runs a full agentic loop internally:
361
+ 1. Sends the user prompt to the model.
362
+ 2. Executes any tool calls (bash / read_file / write_file) locally.
363
+ 3. Feeds results back and loops until the model stops calling tools.
364
+ 4. Yields AgentMessage objects throughout so the controller / TUI see
365
+ a live stream of TEXT, TOOL_START, TOOL_RESULT, and RESULT events.
366
+
367
+ Required env vars:
368
+ OPENROUTER_API_KEY — your OpenRouter API key
369
+ MODEL — model identifier (e.g. "anthropic/claude-3-5-sonnet")
370
+ """
371
+
372
+ def __init__(
373
+ self,
374
+ working_directory: str,
375
+ system_prompt: str,
376
+ model: str,
377
+ ):
378
+ self._cwd = working_directory
379
+ self._system_prompt = system_prompt
380
+ self._model = model
381
+ self._client: Any = None # openai.AsyncOpenAI
382
+ self._messages: list[dict[str, Any]] = []
383
+ # Tool schemas actually sent on the wire. Set in connect() to either
384
+ # the shared _TOOLS or a cache-flagged copy (Claude models). Default
385
+ # keeps the attribute valid even if a caller skips connect().
386
+ self._tools: list[dict[str, Any]] = _TOOLS
387
+ self._pending_query: str | None = None
388
+ # Generate a short session ID from a UUID fragment
389
+ import uuid
390
+
391
+ self._session_id: str = str(uuid.uuid4())[:8]
392
+
393
+ # --- Component A + B state ------------------------------------
394
+ # Persistent HTTP session for the http_request tool. Owns cookies,
395
+ # default headers, and the CSRF token jar across calls.
396
+ self._http: Any = None # httpx.AsyncClient, lazily imported in connect()
397
+ self._csrf_jar: dict[str, str] = {} # form_action_url -> last seen token
398
+ self._http_history: list[dict[str, Any]] = [] # (method, url, status)
399
+ from agent.parsing.html_distiller import HTMLDistiller
400
+
401
+ self._distiller = HTMLDistiller()
402
+
403
+ # ------------------------------------------------------------------
404
+ # AgentBackend interface
405
+ # ------------------------------------------------------------------
406
+
407
+ async def connect(self) -> None:
408
+ """Initialise the AsyncOpenAI client pointed at OpenRouter.
409
+
410
+ Model selection priority: CLI --model > MODEL env var > hardcoded default.
411
+ The caller (controller) is responsible for resolving that priority into
412
+ ``self._model`` via ShadowCatConfig; this method must NOT re-read the
413
+ env var, or it will silently override the CLI flag.
414
+ """
415
+ try:
416
+ from openai import AsyncOpenAI
417
+ except ImportError as exc:
418
+ raise RuntimeError(
419
+ "The 'openai' package is required for the OpenRouter backend. "
420
+ "Install with: uv add openai"
421
+ ) from exc
422
+
423
+ api_key = os.getenv("OPENROUTER_API_KEY")
424
+ if not api_key:
425
+ raise RuntimeError(
426
+ "OPENROUTER_API_KEY environment variable is not set. "
427
+ "Get a key at https://openrouter.ai/keys and add it to your .env file."
428
+ )
429
+
430
+ if not self._model:
431
+ raise RuntimeError(
432
+ "No model specified. Pass --model on the CLI or set MODEL in .env "
433
+ "(e.g. MODEL=qwen/qwen-max). Any OpenRouter model slug is accepted."
434
+ )
435
+
436
+ try:
437
+ # `timeout` here goes through the SDK into its underlying httpx
438
+ # client. Without it, a hung response (e.g. an oversized payload
439
+ # the upstream provider quietly stalls on) keeps the connection
440
+ # open indefinitely — that's the "Thinking… Connecting…" hang
441
+ # the operator sees with `--file <huge>`.
442
+ self._client = AsyncOpenAI(
443
+ base_url="https://openrouter.ai/api/v1",
444
+ api_key=api_key,
445
+ timeout=_LLM_REQUEST_TIMEOUT_SECONDS,
446
+ max_retries=1, # SDK default is 2; we'd rather fail fast and let the controller decide
447
+ )
448
+ except Exception as exc:
449
+ raise RuntimeError(
450
+ f"Failed to initialise OpenRouter client for model '{self._model}': {exc}"
451
+ ) from exc
452
+
453
+ # Spin up the persistent HTTP session used by the http_request tool.
454
+ self._init_http_session()
455
+
456
+ # Mark the static prefix (system prompt + tools) cacheable for Claude
457
+ # models; other providers get the plain, byte-stable form they can
458
+ # auto-cache. See _build_cached_tools / _cacheable_system_message.
459
+ self._tools = _build_cached_tools(_TOOLS, self._model)
460
+ if _is_anthropic_model(self._model):
461
+ self._messages = [_cacheable_system_message(self._system_prompt)]
462
+ else:
463
+ self._messages = [{"role": "system", "content": self._system_prompt}]
464
+
465
+ def _init_http_session(self) -> None:
466
+ """Initialise the persistent httpx session used by the http_request tool.
467
+
468
+ Extracted out of ``connect()`` so subclasses (AnthropicBackend) can
469
+ reuse the same HTTP runtime without re-implementing it.
470
+ Proxy support: set SHADOWCAT_PROXY=http://127.0.0.1:8080 to route
471
+ every agent-issued HTTP request through Burp/mitmproxy.
472
+ """
473
+ self._http = get_shared_http_client()
474
+ self._csrf_jar = get_shared_csrf_jar()
475
+
476
+ async def _close_http_session(self) -> None:
477
+ """Counterpart to _init_http_session. Safe to call when already closed."""
478
+ if self._http is not None:
479
+ await self._http.aclose()
480
+ self._http = None
481
+
482
+ async def disconnect(self) -> None:
483
+ """Close the HTTP client."""
484
+ if self._client is not None:
485
+ await self._client.close()
486
+ self._client = None
487
+ await self._close_http_session()
488
+
489
+ async def query(self, prompt: str) -> None:
490
+ """Queue a user prompt to be sent on the next receive_messages iteration."""
491
+ self._pending_query = prompt
492
+
493
+ async def receive_messages(self) -> AsyncIterator[AgentMessage]: # type: ignore[override]
494
+ """
495
+ Async generator that drives the agentic loop and yields AgentMessages.
496
+
497
+ Must be called after connect() and query().
498
+ Additional query() calls mid-iteration inject new user turns at the
499
+ next safe loop boundary (after tool results are delivered).
500
+ """
501
+ if self._client is None:
502
+ raise RuntimeError("Backend not connected — call connect() first.")
503
+ if self._pending_query is None:
504
+ return
505
+
506
+ self._messages.append({"role": "user", "content": self._pending_query})
507
+ self._pending_query = None
508
+
509
+ total_cost: float = 0.0
510
+ turn = 0
511
+
512
+ while True:
513
+ turn += 1
514
+ if turn > _MAX_AGENT_TURNS:
515
+ yield AgentMessage(
516
+ type=MessageType.ERROR,
517
+ content=(
518
+ f"Agent reached max turn cap ({_MAX_AGENT_TURNS}) without "
519
+ "completing. Stopping to avoid runaway cost."
520
+ ),
521
+ )
522
+ break
523
+ await self._maybe_compact()
524
+ try:
525
+ # Hard ceiling on the round-trip. `asyncio.wait_for` cancels
526
+ # the underlying coroutine on timeout — httpx propagates the
527
+ # cancellation cleanly and the socket is closed. This is the
528
+ # backstop in case the SDK's own httpx timeout doesn't fire
529
+ # (e.g. the provider streams keepalive bytes without ever
530
+ # delivering a complete response).
531
+ with span(f"LLM turn {turn} [{self._model}]"):
532
+ response = await asyncio.wait_for(
533
+ self._client.chat.completions.create(
534
+ model=self._model,
535
+ messages=self._messages,
536
+ tools=self._tools,
537
+ tool_choice="auto",
538
+ # Ask OpenRouter to include cost in the usage block.
539
+ # Without this, `usage.cost` is never populated and the
540
+ # cost counter stays at $0.0000 for the whole session.
541
+ extra_body={"usage": {"include": True}},
542
+ ),
543
+ timeout=_LLM_WAITFOR_TIMEOUT_SECONDS,
544
+ )
545
+ except TimeoutError:
546
+ yield AgentMessage(
547
+ type=MessageType.ERROR,
548
+ content=(
549
+ f"OpenRouter API call timed out after "
550
+ f"{_LLM_WAITFOR_TIMEOUT_SECONDS}s. Most likely cause: "
551
+ f"the prompt is too large for model '{self._model}' "
552
+ f"(check --file size) or the provider is overloaded."
553
+ ),
554
+ )
555
+ return
556
+ except Exception as exc:
557
+ # Surface auth errors, rate limits, and context-overflow
558
+ # errors as a clean ERROR message instead of letting them
559
+ # bubble up as a stack trace through the spinner.
560
+ yield AgentMessage(
561
+ type=MessageType.ERROR,
562
+ content=f"OpenRouter API call failed: {exc.__class__.__name__}: {exc}",
563
+ )
564
+ return
565
+
566
+ choice = response.choices[0]
567
+ msg = choice.message
568
+
569
+ # Accumulate cost when the provider reports it.
570
+ # OpenRouter returns `cost` (USD, float) on the usage object when
571
+ # the request opted in via extra_body. The OpenAI SDK doesn't
572
+ # model this field, so it lands in `model_extra` (pydantic v2)
573
+ # — fall back to that if the attribute isn't directly accessible.
574
+ if response.usage:
575
+ usage = response.usage
576
+ cost_val = getattr(usage, "cost", None)
577
+ if cost_val is None:
578
+ extra = getattr(usage, "model_extra", None) or {}
579
+ cost_val = extra.get("cost")
580
+ total_cost += float(cost_val or 0)
581
+
582
+ # Store a clean assistant turn in history
583
+ asst_entry: dict[str, Any] = {"role": "assistant"}
584
+ if msg.content is not None:
585
+ asst_entry["content"] = msg.content
586
+ if msg.tool_calls:
587
+ asst_entry["tool_calls"] = [
588
+ {
589
+ "id": tc.id,
590
+ "type": "function",
591
+ "function": {
592
+ "name": tc.function.name,
593
+ "arguments": tc.function.arguments,
594
+ },
595
+ }
596
+ for tc in msg.tool_calls
597
+ ]
598
+ self._messages.append(asst_entry)
599
+
600
+ # Yield any text content
601
+ if msg.content:
602
+ yield AgentMessage(type=MessageType.TEXT, content=msg.content)
603
+
604
+ # If no tool calls: check for injected query or finish
605
+ if not msg.tool_calls:
606
+ if self._pending_query:
607
+ self._messages.append({"role": "user", "content": self._pending_query})
608
+ self._pending_query = None
609
+ continue
610
+ break
611
+
612
+ # Execute tool calls concurrently. The model can emit several in a
613
+ # single turn (e.g. parallel recon); running them with
614
+ # asyncio.gather cuts wall-clock latency versus awaiting each in
615
+ # turn. TOOL_START events fire up front, then results are emitted
616
+ # and appended to history in the ORIGINAL order so the
617
+ # (tool_call_id -> result) pairing the API requires stays exact.
618
+ parsed_calls: list[tuple[Any, str, dict[str, Any]]] = []
619
+ for tc in msg.tool_calls:
620
+ name = tc.function.name
621
+ try:
622
+ args = json.loads(tc.function.arguments)
623
+ except json.JSONDecodeError:
624
+ args = {}
625
+ parsed_calls.append((tc, name, args))
626
+ yield AgentMessage(
627
+ type=MessageType.TOOL_START,
628
+ content=None,
629
+ tool_name=name,
630
+ tool_args=args,
631
+ )
632
+
633
+ with span(f"tools turn {turn} [{len(parsed_calls)} call(s)]"):
634
+ tool_outputs = await asyncio.gather(
635
+ *(self._execute_tool(name, args) for _, name, args in parsed_calls),
636
+ return_exceptions=True,
637
+ )
638
+
639
+ for (tc, name, _args), result in zip(parsed_calls, tool_outputs, strict=True):
640
+ if isinstance(result, BaseException):
641
+ result = f"Tool '{name}' raised: {result!r}"
642
+ # Wrap untrusted tool output to prevent injection attacks
643
+ wrapped_result = wrap_untrusted(result, source=name)
644
+
645
+ yield AgentMessage(
646
+ type=MessageType.TOOL_RESULT,
647
+ content=wrapped_result,
648
+ tool_name=name,
649
+ )
650
+
651
+ self._messages.append(
652
+ {
653
+ "role": "tool",
654
+ "tool_call_id": tc.id,
655
+ "content": wrapped_result,
656
+ }
657
+ )
658
+
659
+ # Pick up any instruction injected during tool execution
660
+ if self._pending_query:
661
+ self._messages.append({"role": "user", "content": self._pending_query})
662
+ self._pending_query = None
663
+
664
+ yield AgentMessage(
665
+ type=MessageType.RESULT,
666
+ content=None,
667
+ metadata={"cost_usd": total_cost},
668
+ )
669
+
670
+ @property
671
+ def session_id(self) -> str | None:
672
+ return self._session_id
673
+
674
+ @property
675
+ def supports_resume(self) -> bool:
676
+ # Conversation history lives in memory; no server-side session to resume.
677
+ return False
678
+
679
+ async def resume(self, session_id: str) -> bool:
680
+ return False
681
+
682
+ async def _maybe_compact(self) -> None:
683
+ """Compact conversation history to prevent token bloat."""
684
+ # Cheap gate first: fewer than 8 messages can never compact (we keep
685
+ # system + 6 tail), so skip the expensive json.dumps on every turn.
686
+ if len(self._messages) <= 7:
687
+ return
688
+
689
+ try:
690
+ history_str = json.dumps(self._messages)
691
+ except TypeError:
692
+ history_str = str(self._messages)
693
+
694
+ if len(history_str) < 80000:
695
+ return
696
+
697
+ head = self._messages[0:1] # System prompt
698
+ middle = self._messages[1:-6]
699
+ tail = self._messages[-6:]
700
+
701
+ middle_text = json.dumps(middle)
702
+ summary_prompt = (
703
+ "Summarize the following pentesting context. Keep crucial facts, discovered URLs, "
704
+ "vulnerabilities, and credentials. Be concise.\n\n" + middle_text[:50000]
705
+ )
706
+
707
+ try:
708
+ response = await asyncio.wait_for(
709
+ self._client.chat.completions.create(
710
+ model=self._model,
711
+ messages=[{"role": "user", "content": summary_prompt}],
712
+ ),
713
+ timeout=60,
714
+ )
715
+ summary = response.choices[0].message.content or ""
716
+ except Exception as e:
717
+ summary = f"Context compacted. Summary failed: {e}"
718
+
719
+ compacted_msg = {"role": "user", "content": f"<COMPACTED PRIOR CONTEXT>\n{summary}"}
720
+
721
+ self._messages = head + [compacted_msg] + tail
722
+
723
+ # ------------------------------------------------------------------
724
+ # Tool execution
725
+ # ------------------------------------------------------------------
726
+
727
+ async def _execute_tool(self, name: str, args: dict[str, Any]) -> str:
728
+ if name == "bash":
729
+ return await self._run_bash(args.get("command", ""))
730
+ if name == "read_file":
731
+ return self._read_file(args.get("path", ""))
732
+ if name == "write_file":
733
+ return self._write_file(args.get("path", ""), args.get("content", ""))
734
+ if name == "http_request":
735
+ return await self._send_http_request(args)
736
+ if name == "hash_crack":
737
+ return await self._hash_crack(args.get("hash_value", ""))
738
+ if name == "decode_chain":
739
+ return self._decode_chain(args.get("encoded_string", ""))
740
+ if name == "sqli_probe":
741
+ return await self._sqli_probe(args.get("url", ""))
742
+ return f"Unknown tool: {name}"
743
+
744
+ async def _hash_crack(self, hash_value: str) -> str:
745
+ quoted = shlex.quote(hash_value)
746
+ cmd = (
747
+ f"echo {quoted} > /tmp/target_hash.txt && "
748
+ "hashcat /tmp/target_hash.txt /usr/share/wordlists/rockyou.txt --potfile-disable --quiet || "
749
+ "john /tmp/target_hash.txt --wordlist=/usr/share/wordlists/rockyou.txt"
750
+ )
751
+ return await self._run_bash(cmd)
752
+
753
+ def _decode_chain(self, encoded_string: str) -> str:
754
+ import base64
755
+ import urllib.parse
756
+
757
+ current = encoded_string
758
+ decoded_path = []
759
+ for _ in range(10): # Max depth
760
+ changed = False
761
+ url_dec = urllib.parse.unquote(current)
762
+ if url_dec != current:
763
+ current = url_dec
764
+ decoded_path.append("url")
765
+ changed = True
766
+ continue
767
+ try:
768
+ hex_dec = bytes.fromhex(current).decode("utf-8")
769
+ if hex_dec != current and hex_dec.isprintable():
770
+ current = hex_dec
771
+ decoded_path.append("hex")
772
+ changed = True
773
+ continue
774
+ except Exception:
775
+ pass
776
+ try:
777
+ padded = current + "=" * ((4 - len(current) % 4) % 4)
778
+ b64_dec = base64.b64decode(padded).decode("utf-8")
779
+ if b64_dec != current and b64_dec.isprintable():
780
+ current = b64_dec
781
+ decoded_path.append("base64")
782
+ changed = True
783
+ continue
784
+ except Exception:
785
+ pass
786
+
787
+ if not changed:
788
+ break
789
+
790
+ path_str = " -> ".join(decoded_path) if decoded_path else "none"
791
+ return f"Decoded ({path_str}): {current}"
792
+
793
+ async def _sqli_probe(self, url: str) -> str:
794
+ cmd = f"sqlmap -u {shlex.quote(url)} --batch --risk=1 --level=1"
795
+ return await self._run_bash(cmd)
796
+
797
+ async def _run_bash(self, command: str) -> str:
798
+ # Profiling: surface slow/hung shell tools (nmap, sqlmap, hashcat) in
799
+ # the debug log with the exact command + wall-clock, so "the agent is
800
+ # stuck" is attributable to a specific subprocess rather than guessed at.
801
+ t0 = time.perf_counter()
802
+ preview = command if len(command) <= 80 else command[:79] + "…"
803
+ try:
804
+ proc = await asyncio.create_subprocess_shell(
805
+ command,
806
+ stdout=asyncio.subprocess.PIPE,
807
+ stderr=asyncio.subprocess.STDOUT,
808
+ cwd=self._cwd,
809
+ # Drop LLM-provider API keys before they reach the child so a
810
+ # model-authored command can't exfiltrate them via `env`/`curl`.
811
+ env=scrubbed_env(),
812
+ )
813
+ stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=_TOOL_TIMEOUT_SECONDS)
814
+ output = stdout.decode("utf-8", errors="replace")
815
+ mark(f"bash done in {time.perf_counter() - t0:.2f}s: {preview!r}")
816
+ if len(output) > _OUTPUT_TRUNCATE_CHARS:
817
+ output = output[:_OUTPUT_TRUNCATE_CHARS] + "\n...[output truncated]..."
818
+ return output or "(no output)"
819
+ except TimeoutError:
820
+ mark(f"bash TIMEOUT after {_TOOL_TIMEOUT_SECONDS}s: {preview!r}")
821
+ return f"Command timed out after {_TOOL_TIMEOUT_SECONDS} seconds."
822
+ except Exception as exc:
823
+ mark(f"bash error after {time.perf_counter() - t0:.2f}s: {preview!r}")
824
+ return f"Error running command: {exc}"
825
+
826
+ def _read_file(self, path: str) -> str:
827
+ try:
828
+ full = Path(path) if Path(path).is_absolute() else Path(self._cwd) / path
829
+ content = full.read_text(encoding="utf-8", errors="replace")
830
+ if len(content) > _OUTPUT_TRUNCATE_CHARS:
831
+ content = content[:_OUTPUT_TRUNCATE_CHARS] + "\n...[file truncated]..."
832
+ return content
833
+ except Exception as exc:
834
+ return f"Error reading file: {exc}"
835
+
836
+ def _write_file(self, path: str, content: str) -> str:
837
+ try:
838
+ full = Path(path) if Path(path).is_absolute() else Path(self._cwd) / path
839
+ full.parent.mkdir(parents=True, exist_ok=True)
840
+ full.write_text(content, encoding="utf-8")
841
+ return f"Wrote {len(content)} bytes to {full}."
842
+ except Exception as exc:
843
+ return f"Error writing file: {exc}"
844
+
845
+ # ------------------------------------------------------------------
846
+ # http_request (Component A) + distillation (Component B) executor
847
+ # ------------------------------------------------------------------
848
+
849
+ # Response headers worth showing the LLM. Excludes the high-noise /
850
+ # low-value ones (Date, Connection, Cache-Control, Vary, Set-Cookie which
851
+ # is already encoded in `cookies_set`, Content-Length encoded in body_length).
852
+ _INTERESTING_HEADER_PREFIXES = (
853
+ "content-type",
854
+ "location",
855
+ "server",
856
+ "x-",
857
+ "www-authenticate",
858
+ "access-control",
859
+ "content-security-policy",
860
+ "strict-transport-security",
861
+ "set-cookie", # we keep names in cookies_set but the full directive can leak Path/Secure flags
862
+ "refresh",
863
+ "link",
864
+ "etag",
865
+ "last-modified",
866
+ )
867
+ _NON_HTML_BODY_TRUNCATE = 4000
868
+
869
+ async def _send_http_request(self, args: dict[str, Any]) -> str:
870
+ """Execute one HTTP request via the persistent session.
871
+
872
+ Returns a JSON string (matches the convention used by the other tools
873
+ — the renderer in interface/main.py already decodes and pretty-prints
874
+ nested JSON tool results).
875
+ """
876
+ try:
877
+ import httpx
878
+ except ImportError:
879
+ return json.dumps({"error": "httpx not installed in container"})
880
+
881
+ if self._http is None:
882
+ return json.dumps(
883
+ {"error": "HTTP session not initialised — backend.connect() not called"}
884
+ )
885
+
886
+ method = (args.get("method") or "GET").upper()
887
+ url = args.get("url") or ""
888
+ if not url:
889
+ return json.dumps({"error": "missing required parameter: url"})
890
+
891
+ headers = dict(args.get("headers") or {})
892
+ follow = bool(args.get("follow_redirects", True))
893
+ timeout = float(args.get("timeout_seconds") or 30)
894
+
895
+ # Body assembly. Three mutually-exclusive options: form > json_body > body.
896
+ # Tool description tells the LLM they're mutually exclusive; if it
897
+ # passes multiple we honour this priority order and silently ignore
898
+ # the others (httpx would 500 otherwise on conflicting kwargs).
899
+ content: str | None = None
900
+ data: dict[str, Any] | None = None
901
+ json_body: Any = None
902
+ form_arg = args.get("form")
903
+ json_arg = args.get("json_body")
904
+
905
+ if form_arg is not None:
906
+ data = dict(form_arg)
907
+ # Auto-CSRF: if the form action is known and the payload doesn't
908
+ # already carry a token field, inject the cached one.
909
+ if args.get("auto_csrf", True):
910
+ token = self._csrf_jar.get(url)
911
+ if token:
912
+ # Use the first common token name unless the form already
913
+ # has one. This is best-effort; if the framework expects a
914
+ # specific name not in this list, the LLM can pass it
915
+ # explicitly in `form` and we won't clobber.
916
+ has_token_field = any(
917
+ k.lower()
918
+ in {
919
+ "csrf_token",
920
+ "csrf",
921
+ "_token",
922
+ "csrfmiddlewaretoken",
923
+ "authenticity_token",
924
+ "__requestverificationtoken",
925
+ "xsrf_token",
926
+ }
927
+ for k in data
928
+ )
929
+ if not has_token_field:
930
+ data["csrf_token"] = token
931
+ elif json_arg is not None:
932
+ json_body = json_arg
933
+ elif args.get("body") is not None:
934
+ content = args["body"]
935
+
936
+ t0 = time.perf_counter()
937
+ try:
938
+ resp = await self._http.request(
939
+ method,
940
+ url,
941
+ headers=headers,
942
+ content=content,
943
+ data=data,
944
+ json=json_body,
945
+ follow_redirects=follow,
946
+ timeout=timeout,
947
+ )
948
+ except httpx.TimeoutException as e:
949
+ return json.dumps(
950
+ {
951
+ "error": "timeout",
952
+ "detail": str(e),
953
+ "url": url,
954
+ "elapsed_ms": int((time.perf_counter() - t0) * 1000),
955
+ }
956
+ )
957
+ except httpx.RequestError as e:
958
+ return json.dumps(
959
+ {
960
+ "error": e.__class__.__name__,
961
+ "detail": str(e),
962
+ "url": url,
963
+ }
964
+ )
965
+
966
+ elapsed_ms = int((time.perf_counter() - t0) * 1000)
967
+ ctype = (resp.headers.get("content-type") or "").lower()
968
+
969
+ # Body handling — distill HTML, pass JSON through (already structured),
970
+ # truncate other text, mark binary.
971
+ body_payload: Any
972
+ body_kind: str
973
+ if "html" in ctype or (ctype == "" and _looks_like_html(resp.text[:512])):
974
+ body_payload = self._distiller.distill(resp.text, base_url=str(resp.url))
975
+ body_kind = "html-distilled"
976
+ # Harvest CSRF tokens the distiller already extracted.
977
+ for action, token in body_payload.get("csrf_tokens", {}).items():
978
+ self._csrf_jar[action] = token
979
+
980
+ # Optional RAG hook for framework versions found in HTML
981
+ try:
982
+ # Only do RAG when there's actually a framework hint — avoids
983
+ # paying the (singleton) lookup on every HTML response.
984
+ fw = resp.headers.get("x-powered-by", "")
985
+ if fw:
986
+ from agent.rag_module.knowledge_base import get_shared_kb
987
+
988
+ cves = await get_shared_kb().query_framework_cves(fw, "unknown", top_k=2)
989
+ if cves:
990
+ body_payload["rag_cves"] = cves
991
+ except Exception:
992
+ pass
993
+ elif "json" in ctype:
994
+ body_payload = resp.text # already structured, leave raw
995
+ body_kind = "json"
996
+ elif _is_textual(ctype):
997
+ body_text = resp.text
998
+ truncated = len(body_text) > self._NON_HTML_BODY_TRUNCATE
999
+ body_payload = body_text[: self._NON_HTML_BODY_TRUNCATE]
1000
+ body_kind = "text-truncated" if truncated else "text"
1001
+ else:
1002
+ body_payload = None
1003
+ body_kind = f"binary ({len(resp.content)} bytes)"
1004
+
1005
+ result: dict[str, Any] = {
1006
+ "status": resp.status_code,
1007
+ "url": str(resp.url),
1008
+ "method": method,
1009
+ "headers": self._filter_headers(resp.headers),
1010
+ "body": body_payload,
1011
+ "body_kind": body_kind,
1012
+ "body_length_original": len(resp.content),
1013
+ "cookies_set": [c.name for c in resp.cookies.jar],
1014
+ "elapsed_ms": elapsed_ms,
1015
+ "redirect_chain": [str(h.url) for h in resp.history] or None,
1016
+ "summary_only": True, # tells LLM this is distilled — request "body" via direct curl if needed
1017
+ }
1018
+
1019
+ self._http_history.append({"method": method, "url": url, "status": resp.status_code})
1020
+ return json.dumps(result, separators=(",", ":"), default=str)
1021
+
1022
+ def _filter_headers(self, headers: Any) -> dict[str, str]:
1023
+ """Keep only headers likely to carry pentest signal."""
1024
+ out: dict[str, str] = {}
1025
+ for k, v in headers.items():
1026
+ lk = k.lower()
1027
+ if any(lk.startswith(p) for p in self._INTERESTING_HEADER_PREFIXES):
1028
+ # Cap header value length (CSP and Permissions-Policy can be huge)
1029
+ out[k] = v if len(v) < 400 else v[:400] + "…"
1030
+ return out
1031
+
1032
+
1033
+ # Module-level helpers (kept out of the class — pure functions, no state).
1034
+ def _is_textual(content_type: str) -> bool:
1035
+ if not content_type:
1036
+ return True # if server didn't say, assume text and try
1037
+ lower = content_type.lower()
1038
+ return any(
1039
+ s in lower
1040
+ for s in (
1041
+ "text/",
1042
+ "javascript",
1043
+ "json",
1044
+ "xml",
1045
+ "x-www-form-urlencoded",
1046
+ )
1047
+ )
1048
+
1049
+
1050
+ def _looks_like_html(snippet: str) -> bool:
1051
+ s = snippet.lstrip().lower()
1052
+ return s.startswith(("<!doctype", "<html", "<head", "<!--", "<body"))
1053
+
1054
+
1055
+ # ============================================================================
1056
+ # AnthropicBackend — direct api.anthropic.com auth via ANTHROPIC_API_KEY
1057
+ # ============================================================================
1058
+ #
1059
+ # Inherits from OpenRouterBackend to reuse ALL tool execution logic
1060
+ # (bash / read_file / write_file / http_request / HTML distillation / CSRF
1061
+ # jar / proxy support). Only the LLM-talking surface (connect, disconnect,
1062
+ # query, receive_messages) is overridden because Anthropic's wire protocol
1063
+ # differs from OpenAI's:
1064
+ #
1065
+ # - tools format: {"name", "description", "input_schema"} — no "function" wrapper
1066
+ # - system prompt: top-level `system=` parameter, not a message
1067
+ # - tool calls: ToolUseBlock inside response.content list (not tool_calls field)
1068
+ # - tool results: nested in a SINGLE user message's content array
1069
+ # - max_tokens: REQUIRED parameter, no default
1070
+ # - cost: usage.input_tokens + usage.output_tokens (no USD figure returned)
1071
+
1072
+
1073
+ # Best-effort USD pricing for cost reporting. These are correct as of my
1074
+ # training cutoff and WILL go stale. If you see $0.0000 on a successful
1075
+ # session, your model isn't in this table — add it or accept token-only
1076
+ # accounting. Prices are USD per 1M tokens.
1077
+ _ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
1078
+ "claude-3-5-sonnet-20241022": {"input": 3.0, "output": 15.0},
1079
+ "claude-3-5-sonnet-latest": {"input": 3.0, "output": 15.0},
1080
+ "claude-3-5-haiku-20241022": {"input": 0.80, "output": 4.0},
1081
+ "claude-3-5-haiku-latest": {"input": 0.80, "output": 4.0},
1082
+ "claude-3-opus-20240229": {"input": 15.0, "output": 75.0},
1083
+ "claude-sonnet-4-20250514": {"input": 3.0, "output": 15.0},
1084
+ "claude-sonnet-4-5": {"input": 3.0, "output": 15.0},
1085
+ "claude-opus-4-20250514": {"input": 15.0, "output": 75.0},
1086
+ }
1087
+
1088
+
1089
+ def _anthropic_tools_from_openai_schema(openai_tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
1090
+ """Convert _TOOLS (OpenAI function-calling format) to Anthropic format.
1091
+
1092
+ OpenAI: {"type":"function","function":{"name","description","parameters"}}
1093
+ Anthropic: {"name","description","input_schema"}
1094
+ """
1095
+ out: list[dict[str, Any]] = []
1096
+ for t in openai_tools:
1097
+ if t.get("type") != "function":
1098
+ continue
1099
+ fn = t["function"]
1100
+ out.append(
1101
+ {
1102
+ "name": fn["name"],
1103
+ "description": fn.get("description", ""),
1104
+ "input_schema": fn.get("parameters", {"type": "object", "properties": {}}),
1105
+ }
1106
+ )
1107
+ return out
1108
+
1109
+
1110
+ def _cacheable_anthropic_system(system_prompt: str) -> list[dict[str, Any]]:
1111
+ """Anthropic ``system`` param as a single cache-flagged text block.
1112
+
1113
+ Anthropic accepts ``system`` as a string OR a list of content blocks; the
1114
+ list form is the only one that can carry ``cache_control``.
1115
+ """
1116
+ return [
1117
+ {
1118
+ "type": "text",
1119
+ "text": system_prompt,
1120
+ "cache_control": _CACHE_CONTROL_EPHEMERAL,
1121
+ }
1122
+ ]
1123
+
1124
+
1125
+ class AnthropicBackend(OpenRouterBackend):
1126
+ """Talks to api.anthropic.com directly with an ANTHROPIC_API_KEY.
1127
+
1128
+ Selected by the controller when ``SHADOWCAT_AUTH_MODE`` is ``anthropic``
1129
+ or ``claude_subscription``. Session tokens (sk-ant-sid-...) are not
1130
+ supported by the Anthropic API and were removed from config.sh.
1131
+
1132
+ The reuse-via-inheritance trick: tool execution (bash, read_file,
1133
+ write_file, http_request) and HTTP session management live on the parent
1134
+ class; we override only what's different about the LLM wire protocol.
1135
+ """
1136
+
1137
+ _MAX_TOKENS_PER_RESPONSE = 4096 # Anthropic requires this be set explicitly
1138
+
1139
+ async def connect(self) -> None:
1140
+ """Initialise the Anthropic client and the shared HTTP session."""
1141
+ try:
1142
+ from anthropic import AsyncAnthropic
1143
+ except ImportError as exc:
1144
+ raise RuntimeError(
1145
+ "The 'anthropic' package is required for the AnthropicBackend. "
1146
+ "Install with: uv add anthropic"
1147
+ ) from exc
1148
+
1149
+ api_key = os.getenv("ANTHROPIC_API_KEY")
1150
+ if not api_key:
1151
+ raise RuntimeError(
1152
+ "ANTHROPIC_API_KEY environment variable is not set. "
1153
+ "Get a key at https://console.anthropic.com/settings/keys "
1154
+ "and add it to your .env.auth file via `make config` option [4]."
1155
+ )
1156
+ if not api_key.startswith("sk-ant-api"):
1157
+ # Session tokens (sk-ant-sid-) won't authenticate against the API.
1158
+ # Catch this loudly here rather than letting the user see a generic 401.
1159
+ raise RuntimeError(
1160
+ "ANTHROPIC_API_KEY does not start with 'sk-ant-api'. The "
1161
+ "Anthropic API only accepts API keys, not session tokens. "
1162
+ "If you have a session token (sk-ant-sid-...), it won't "
1163
+ "work here — get a real API key at "
1164
+ "https://console.anthropic.com/settings/keys."
1165
+ )
1166
+ if not self._model:
1167
+ raise RuntimeError(
1168
+ "No model specified. Pass --model on the CLI or set MODEL in "
1169
+ ".env.auth (e.g. MODEL=claude-3-5-sonnet-latest). For the "
1170
+ "Anthropic backend, use plain Claude model names without the "
1171
+ "'anthropic/' prefix that OpenRouter expects."
1172
+ )
1173
+
1174
+ try:
1175
+ # Same rationale as OpenRouterBackend.connect(): a configured
1176
+ # client-level timeout prevents indefinite hangs when the API
1177
+ # stalls mid-request (oversized prompt, network blip, etc.).
1178
+ self._client = AsyncAnthropic(
1179
+ api_key=api_key,
1180
+ timeout=_LLM_REQUEST_TIMEOUT_SECONDS,
1181
+ max_retries=1,
1182
+ )
1183
+ except Exception as exc:
1184
+ raise RuntimeError(
1185
+ f"Failed to initialise Anthropic client for model '{self._model}': {exc}"
1186
+ ) from exc
1187
+
1188
+ # Tool execution runtime (shared with OpenRouterBackend)
1189
+ self._init_http_session()
1190
+
1191
+ # Anthropic does NOT use system messages in the messages list.
1192
+ # The system prompt is a top-level parameter on every call.
1193
+ # Conversation history holds only user/assistant exchanges.
1194
+ self._messages = []
1195
+
1196
+ async def disconnect(self) -> None:
1197
+ """Close both clients."""
1198
+ if self._client is not None:
1199
+ # AsyncAnthropic exposes aclose() in recent SDKs
1200
+ close = getattr(self._client, "aclose", None) or getattr(self._client, "close", None)
1201
+ if close is not None:
1202
+ result = close()
1203
+ if hasattr(result, "__await__"):
1204
+ await result
1205
+ self._client = None
1206
+ await self._close_http_session()
1207
+
1208
+ async def _maybe_compact(self) -> None:
1209
+ """Compact conversation history for Anthropic."""
1210
+ # Cheap gate first: skip the expensive json.dumps unless we might
1211
+ # actually compact.
1212
+ if len(self._messages) <= 6:
1213
+ return
1214
+
1215
+ try:
1216
+ history_str = json.dumps(self._messages, default=str)
1217
+ except TypeError:
1218
+ history_str = str(self._messages)
1219
+
1220
+ if len(history_str) < 80000:
1221
+ return
1222
+
1223
+ middle = self._messages[:-6]
1224
+ tail = self._messages[-6:]
1225
+
1226
+ middle_text = str(middle)
1227
+ summary_prompt = (
1228
+ "Summarize the following pentesting context. Keep crucial facts, discovered URLs, "
1229
+ "vulnerabilities, and credentials. Be concise.\n\n" + middle_text[:50000]
1230
+ )
1231
+
1232
+ try:
1233
+ response = await asyncio.wait_for(
1234
+ self._client.messages.create(
1235
+ model=self._model,
1236
+ max_tokens=self._MAX_TOKENS_PER_RESPONSE,
1237
+ system="You are a helpful summarization assistant.",
1238
+ messages=[{"role": "user", "content": summary_prompt}],
1239
+ ),
1240
+ timeout=60,
1241
+ )
1242
+ summary = response.content[0].text if response.content else ""
1243
+ except Exception as e:
1244
+ summary = f"Context compacted. Summary failed: {e}"
1245
+
1246
+ compacted_msg = {"role": "user", "content": f"<COMPACTED PRIOR CONTEXT>\n{summary}"}
1247
+
1248
+ self._messages = [compacted_msg] + tail
1249
+
1250
+ async def receive_messages(self) -> AsyncIterator[AgentMessage]: # type: ignore[override]
1251
+ """Drive the agentic loop using Anthropic's message API.
1252
+
1253
+ Differences from the parent implementation:
1254
+ - System prompt is a top-level kwarg, not a message
1255
+ - Tool calls live in response.content as ToolUseBlock instances
1256
+ - Tool results go in a SINGLE user message's content array (not
1257
+ separate messages per tool, the way OpenAI does it)
1258
+ - Loop terminates when stop_reason != "tool_use"
1259
+ """
1260
+ if self._client is None:
1261
+ raise RuntimeError("Backend not connected — call connect() first.")
1262
+ if self._pending_query is None:
1263
+ return
1264
+
1265
+ self._messages.append({"role": "user", "content": self._pending_query})
1266
+ self._pending_query = None
1267
+
1268
+ anthropic_tools = _anthropic_tools_from_openai_schema(_TOOLS)
1269
+ # Cache breakpoint on the last tool → the whole (stable) tool block is
1270
+ # cached. The system block is cached separately in the create() call
1271
+ # below, so the cached prefix spans tools + system on every turn.
1272
+ if anthropic_tools:
1273
+ anthropic_tools[-1] = {
1274
+ **anthropic_tools[-1],
1275
+ "cache_control": _CACHE_CONTROL_EPHEMERAL,
1276
+ }
1277
+ pricing = _ANTHROPIC_PRICING.get(self._model, {})
1278
+ total_cost: float = 0.0
1279
+ total_in_tokens = 0
1280
+ total_out_tokens = 0
1281
+ turn = 0
1282
+
1283
+ while True:
1284
+ turn += 1
1285
+ if turn > _MAX_AGENT_TURNS:
1286
+ yield AgentMessage(
1287
+ type=MessageType.ERROR,
1288
+ content=(
1289
+ f"Agent reached max turn cap ({_MAX_AGENT_TURNS}) without "
1290
+ "completing. Stopping to avoid runaway cost."
1291
+ ),
1292
+ )
1293
+ break
1294
+ await self._maybe_compact()
1295
+ try:
1296
+ with span(f"LLM turn {turn} [{self._model}]"):
1297
+ response = await asyncio.wait_for(
1298
+ self._client.messages.create(
1299
+ model=self._model,
1300
+ max_tokens=self._MAX_TOKENS_PER_RESPONSE,
1301
+ system=_cacheable_anthropic_system(self._system_prompt),
1302
+ messages=self._messages,
1303
+ tools=anthropic_tools,
1304
+ ),
1305
+ timeout=_LLM_WAITFOR_TIMEOUT_SECONDS,
1306
+ )
1307
+ except TimeoutError:
1308
+ yield AgentMessage(
1309
+ type=MessageType.ERROR,
1310
+ content=(
1311
+ f"Anthropic API call timed out after "
1312
+ f"{_LLM_WAITFOR_TIMEOUT_SECONDS}s. Most likely cause: "
1313
+ f"the prompt is too large for model '{self._model}' "
1314
+ f"(check --file size) or the provider is overloaded."
1315
+ ),
1316
+ )
1317
+ return
1318
+ except Exception as exc:
1319
+ # Surface auth errors and rate limits clearly to the UI layer
1320
+ yield AgentMessage(
1321
+ type=MessageType.ERROR,
1322
+ content=f"Anthropic API call failed: {exc.__class__.__name__}: {exc}",
1323
+ )
1324
+ return
1325
+
1326
+ # Accumulate token + cost accounting
1327
+ if response.usage:
1328
+ in_t = response.usage.input_tokens
1329
+ out_t = response.usage.output_tokens
1330
+ total_in_tokens += in_t
1331
+ total_out_tokens += out_t
1332
+ if pricing:
1333
+ total_cost += (
1334
+ in_t * pricing["input"] / 1_000_000 + out_t * pricing["output"] / 1_000_000
1335
+ )
1336
+
1337
+ # Persist the assistant turn — content blocks must survive
1338
+ # round-trip back to Anthropic. The SDK objects are pydantic
1339
+ # models; we serialise via model_dump for storage safety.
1340
+ asst_content_dumped = [
1341
+ block.model_dump() if hasattr(block, "model_dump") else block
1342
+ for block in response.content
1343
+ ]
1344
+ self._messages.append({"role": "assistant", "content": asst_content_dumped})
1345
+
1346
+ # Emit text blocks in order, then run all tool_use blocks
1347
+ # concurrently. Anthropic interleaves text and tool_use in
1348
+ # response.content; text is surfaced immediately while the tool
1349
+ # calls fan out via asyncio.gather. Results are paired back by
1350
+ # tool_use_id and appended in original order.
1351
+ tool_results: list[dict[str, Any]] = []
1352
+ tool_uses: list[Any] = []
1353
+ for block in response.content:
1354
+ btype = getattr(block, "type", None)
1355
+
1356
+ if btype == "text":
1357
+ text = getattr(block, "text", "") or ""
1358
+ if text:
1359
+ yield AgentMessage(type=MessageType.TEXT, content=text)
1360
+
1361
+ elif btype == "tool_use":
1362
+ name = block.name
1363
+ args = block.input if isinstance(block.input, dict) else {}
1364
+ tool_uses.append(block)
1365
+
1366
+ yield AgentMessage(
1367
+ type=MessageType.TOOL_START,
1368
+ content=None,
1369
+ tool_name=name,
1370
+ tool_args=args,
1371
+ )
1372
+
1373
+ if tool_uses:
1374
+ with span(f"tools turn {turn} [{len(tool_uses)} call(s)]"):
1375
+ tool_outputs = await asyncio.gather(
1376
+ *(
1377
+ self._execute_tool(b.name, b.input if isinstance(b.input, dict) else {})
1378
+ for b in tool_uses
1379
+ ),
1380
+ return_exceptions=True,
1381
+ )
1382
+
1383
+ for block, result in zip(tool_uses, tool_outputs, strict=True):
1384
+ if isinstance(result, BaseException):
1385
+ result = f"Tool '{block.name}' raised: {result!r}"
1386
+ # Wrap untrusted tool output to prevent injection attacks
1387
+ wrapped_result = wrap_untrusted(result, source=block.name)
1388
+
1389
+ yield AgentMessage(
1390
+ type=MessageType.TOOL_RESULT,
1391
+ content=wrapped_result,
1392
+ tool_name=block.name,
1393
+ )
1394
+
1395
+ tool_results.append(
1396
+ {
1397
+ "type": "tool_result",
1398
+ "tool_use_id": block.id,
1399
+ "content": wrapped_result,
1400
+ }
1401
+ )
1402
+
1403
+ # Continuation decision — Anthropic tells us via stop_reason
1404
+ if response.stop_reason == "tool_use" and tool_results:
1405
+ # ALL tool results go in ONE user message — this is the
1406
+ # structural difference from OpenAI's per-result messages.
1407
+ self._messages.append({"role": "user", "content": tool_results})
1408
+
1409
+ # Inject any user query that arrived during tool execution
1410
+ if self._pending_query:
1411
+ self._messages.append({"role": "user", "content": self._pending_query})
1412
+ self._pending_query = None
1413
+ continue
1414
+
1415
+ # No tools called this turn — check for an injected user message
1416
+ if self._pending_query:
1417
+ self._messages.append({"role": "user", "content": self._pending_query})
1418
+ self._pending_query = None
1419
+ continue
1420
+
1421
+ # Conversation turn truly complete
1422
+ break
1423
+
1424
+ yield AgentMessage(
1425
+ type=MessageType.RESULT,
1426
+ content=None,
1427
+ metadata={
1428
+ "cost_usd": total_cost,
1429
+ "input_tokens": total_in_tokens,
1430
+ "output_tokens": total_out_tokens,
1431
+ "model": self._model,
1432
+ "pricing_known": bool(pricing),
1433
+ },
1434
+ )
1435
+
1436
+
1437
+ # ============================================================================
1438
+ # ClaudeSDKBackend — Option 4 (Claude Subscription via OAuth)
1439
+ # ============================================================================
1440
+ #
1441
+ # Wraps `claude-agent-sdk` to drive the agentic loop under the user's
1442
+ # Claude Pro / Team subscription. Auth is handled by the SDK (OAuth via
1443
+ # `claude login` — token cached at ~/.claude/). No API key required.
1444
+ #
1445
+ # Mirrors upstream shadowcat-dast's ClaudeCodeBackend almost line-for-line,
1446
+ # adapted to our AgentBackend interface. Direct subclass of AgentBackend
1447
+ # (NOT OpenRouterBackend) because tool execution, message history, and HTTP
1448
+ # session are all owned by the SDK — there's nothing to inherit.
1449
+ #
1450
+ # Known limitations (documented honestly, not hidden):
1451
+ # - Our custom http_request tool with HTML distillation is NOT available;
1452
+ # the SDK provides its own WebFetch which returns raw HTML. If you need
1453
+ # distillation in this backend, file a follow-up to register http_request
1454
+ # as a custom SDK tool.
1455
+ # - Only TOOL_START events are emitted, never TOOL_RESULT — the SDK runs
1456
+ # tools internally and doesn't expose results to the host. Flag detection
1457
+ # in raw tool output (controller.py TOOL_RESULT branch) will not fire for
1458
+ # this backend; only flags in the agent's narration text are caught.
1459
+
1460
+
1461
+ class ClaudeSDKBackend(AgentBackend):
1462
+ """Drives Claude through the official `claude-agent-sdk` (OAuth-backed).
1463
+
1464
+ Selected by the controller when ``SHADOWCAT_AUTH_MODE=claude_subscription``.
1465
+ """
1466
+
1467
+ def __init__(
1468
+ self,
1469
+ working_directory: str,
1470
+ system_prompt: str,
1471
+ model: str,
1472
+ permission_mode: str = "bypassPermissions",
1473
+ ):
1474
+ self._cwd = working_directory
1475
+ self._system_prompt = system_prompt
1476
+ self._model = model
1477
+ # bypassPermissions skips the SDK's interactive permission prompts —
1478
+ # mandatory for autonomous operation. acceptEdits is a safer alternative
1479
+ # if you want the agent to ask before destructive actions.
1480
+ self._permission_mode = permission_mode
1481
+ self._client: Any = None # claude_agent_sdk.ClaudeSDKClient
1482
+ self._session_id: str | None = None
1483
+
1484
+ # ------------------------------------------------------------------
1485
+ # Auth env-var preparation
1486
+ # ------------------------------------------------------------------
1487
+ #
1488
+ # The SDK picks its auth source automatically based on env vars:
1489
+ # - ANTHROPIC_API_KEY present -> uses that key (metered billing)
1490
+ # - ANTHROPIC_API_KEY absent -> falls back to OAuth from `claude login`
1491
+ # (= Claude Pro/Team subscription, the whole point of this backend)
1492
+ #
1493
+ # So for Option 4 we deliberately clear ANTHROPIC_API_KEY before launching
1494
+ # the SDK, even if the user has one set in their shell. This is the
1495
+ # same trick upstream uses.
1496
+
1497
+ def _build_env_overrides(self) -> dict[str, str]:
1498
+ """Build the env dict to pass to the SDK's subprocess.
1499
+
1500
+ CRITICAL: ClaudeAgentOptions.env= uses Python subprocess semantics —
1501
+ the dict REPLACES the child's environment, it does NOT merge with
1502
+ the parent's. If we passed just {"ANTHROPIC_API_KEY": ""}, the
1503
+ bundled `claude` CLI would launch with no HOME, no XDG_*, no PATH,
1504
+ no USER — and immediately fail to find ~/.claude/.credentials.json,
1505
+ reporting "Not logged in" even though the user IS logged in.
1506
+
1507
+ Fix: copy the full parent env first, THEN apply overrides on top.
1508
+
1509
+ (Upstream shadowcat-dast has the same latent bug — they get away with it
1510
+ because their typical user has ANTHROPIC_API_KEY unset in the shell,
1511
+ so the branch never fires and no env replacement happens.)
1512
+ """
1513
+ env = dict(os.environ)
1514
+ if env.get("ANTHROPIC_API_KEY"):
1515
+ # Force OAuth fallback by clearing the API key for the subprocess.
1516
+ # The SDK treats empty string as "unset" for auth-selection purposes.
1517
+ # We keep all other vars (HOME, PATH, XDG_*, USER, etc.) intact so
1518
+ # the bundled CLI can find its OAuth credentials at $HOME/.claude/.
1519
+ env["ANTHROPIC_API_KEY"] = ""
1520
+ return env
1521
+
1522
+ async def connect(self) -> None:
1523
+ try:
1524
+ from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
1525
+ except ImportError as exc:
1526
+ raise RuntimeError(
1527
+ "The 'claude-agent-sdk' package is required for the Claude "
1528
+ "Subscription backend. Install with: uv add claude-agent-sdk\n"
1529
+ "You also need the Claude Code CLI on PATH: "
1530
+ "npm install -g @anthropic-ai/claude-code"
1531
+ ) from exc
1532
+
1533
+ options = ClaudeAgentOptions(
1534
+ cwd=self._cwd,
1535
+ permission_mode=self._permission_mode, # type: ignore[arg-type]
1536
+ system_prompt=self._system_prompt,
1537
+ model=self._model,
1538
+ env=self._build_env_overrides(),
1539
+ )
1540
+ try:
1541
+ self._client = ClaudeSDKClient(options=options)
1542
+ result = self._client.connect()
1543
+ # Some SDK versions return a coroutine, others return None — handle both.
1544
+ if result is not None:
1545
+ await result
1546
+ except Exception as exc:
1547
+ # The most common failure is "claude login required" or "Claude
1548
+ # Code CLI not found on PATH". Re-raise with a clearer message.
1549
+ raise RuntimeError(
1550
+ f"Failed to start Claude Agent SDK session: {exc}\n"
1551
+ f" - Ensure 'claude' CLI is on PATH (try: claude --version)\n"
1552
+ f" - Ensure you're logged in (run: claude login)\n"
1553
+ f" - Verify your subscription at https://claude.ai/settings"
1554
+ ) from exc
1555
+
1556
+ async def disconnect(self) -> None:
1557
+ if self._client is not None:
1558
+ try:
1559
+ result = self._client.disconnect()
1560
+ if result is not None:
1561
+ await result
1562
+ except Exception:
1563
+ # Best-effort cleanup; suppress to avoid masking earlier errors.
1564
+ pass
1565
+ self._client = None
1566
+
1567
+ async def query(self, prompt: str) -> None:
1568
+ if self._client is None:
1569
+ raise RuntimeError("Backend not connected — call connect() first.")
1570
+ result = self._client.query(prompt)
1571
+ if result is not None:
1572
+ await result
1573
+
1574
+ async def receive_messages(self) -> AsyncIterator[AgentMessage]: # type: ignore[override]
1575
+ """Translate SDK message stream into AgentMessage events.
1576
+
1577
+ We import the SDK message types lazily inside the function so that
1578
+ `import backend` doesn't hard-fail when the SDK isn't installed —
1579
+ users of the other backends shouldn't pay that cost.
1580
+ """
1581
+ try:
1582
+ from claude_agent_sdk import (
1583
+ AssistantMessage,
1584
+ ResultMessage,
1585
+ TextBlock,
1586
+ ToolUseBlock,
1587
+ )
1588
+ except ImportError as exc:
1589
+ raise RuntimeError(
1590
+ "claude-agent-sdk not available — see connect() for install hints."
1591
+ ) from exc
1592
+
1593
+ if self._client is None:
1594
+ raise RuntimeError("Backend not connected — call connect() first.")
1595
+
1596
+ async for msg in self._client.receive_response():
1597
+ if isinstance(msg, AssistantMessage):
1598
+ for block in msg.content:
1599
+ if isinstance(block, TextBlock):
1600
+ yield AgentMessage(
1601
+ type=MessageType.TEXT,
1602
+ content=block.text,
1603
+ )
1604
+ elif isinstance(block, ToolUseBlock):
1605
+ yield AgentMessage(
1606
+ type=MessageType.TOOL_START,
1607
+ content=None,
1608
+ tool_name=block.name,
1609
+ tool_args=block.input if isinstance(block.input, dict) else {},
1610
+ )
1611
+ # NOTE: no matching TOOL_RESULT yield — see class docstring.
1612
+
1613
+ elif isinstance(msg, ResultMessage):
1614
+ yield AgentMessage(
1615
+ type=MessageType.RESULT,
1616
+ content=None,
1617
+ metadata={
1618
+ # The SDK populates this field directly when the upstream
1619
+ # provider reports cost. For OAuth/subscription mode this
1620
+ # is typically 0 because subscription usage isn't billed
1621
+ # per-token to the user.
1622
+ "cost_usd": getattr(msg, "total_cost_usd", 0) or 0,
1623
+ "model": self._model,
1624
+ },
1625
+ )
1626
+
1627
+ @property
1628
+ def session_id(self) -> str | None:
1629
+ return self._session_id
1630
+
1631
+ @property
1632
+ def supports_resume(self) -> bool:
1633
+ # The SDK supports resume natively via the `resume=` option.
1634
+ return True
1635
+
1636
+ async def resume(self, session_id: str) -> bool:
1637
+ """Reconnect with a previous SDK session ID."""
1638
+ try:
1639
+ from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
1640
+ except ImportError as exc:
1641
+ raise RuntimeError("claude-agent-sdk not installed") from exc
1642
+
1643
+ if self._client is not None:
1644
+ try:
1645
+ result = self._client.disconnect()
1646
+ if result is not None:
1647
+ await result
1648
+ except Exception:
1649
+ pass
1650
+
1651
+ options = ClaudeAgentOptions(
1652
+ cwd=self._cwd,
1653
+ permission_mode=self._permission_mode, # type: ignore[arg-type]
1654
+ system_prompt=self._system_prompt,
1655
+ model=self._model,
1656
+ resume=session_id,
1657
+ env=self._build_env_overrides(),
1658
+ )
1659
+ try:
1660
+ self._client = ClaudeSDKClient(options=options)
1661
+ result = self._client.connect()
1662
+ if result is not None:
1663
+ await result
1664
+ except Exception as exc:
1665
+ raise RuntimeError(f"Failed to resume SDK session {session_id}: {exc}") from exc
1666
+ self._session_id = session_id
1667
+ return True