scythe-ttp 0.12.1__py3-none-any.whl → 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scythe-ttp might be problematic. Click here for more details.
- scythe/auth/__init__.py +3 -1
- scythe/auth/base.py +9 -0
- scythe/auth/cookie_jwt.py +172 -0
- scythe/core/headers.py +152 -0
- scythe/journeys/__init__.py +2 -1
- scythe/journeys/actions.py +124 -1
- scythe/journeys/base.py +40 -7
- scythe/journeys/executor.py +63 -22
- scythe/ttps/web/uuid_guessing.py +3 -2
- {scythe_ttp-0.12.1.dist-info → scythe_ttp-0.13.0.dist-info}/METADATA +3 -1
- {scythe_ttp-0.12.1.dist-info → scythe_ttp-0.13.0.dist-info}/RECORD +14 -13
- {scythe_ttp-0.12.1.dist-info → scythe_ttp-0.13.0.dist-info}/WHEEL +0 -0
- {scythe_ttp-0.12.1.dist-info → scythe_ttp-0.13.0.dist-info}/licenses/LICENSE +0 -0
- {scythe_ttp-0.12.1.dist-info → scythe_ttp-0.13.0.dist-info}/top_level.txt +0 -0
scythe/auth/__init__.py
CHANGED
|
@@ -8,9 +8,11 @@ authenticate before executing their main functionality.
|
|
|
8
8
|
from .base import Authentication
|
|
9
9
|
from .bearer import BearerTokenAuth
|
|
10
10
|
from .basic import BasicAuth
|
|
11
|
+
from .cookie_jwt import CookieJWTAuth
|
|
11
12
|
|
|
12
13
|
__all__ = [
|
|
13
14
|
'Authentication',
|
|
14
15
|
'BearerTokenAuth',
|
|
15
|
-
'BasicAuth'
|
|
16
|
+
'BasicAuth',
|
|
17
|
+
'CookieJWTAuth',
|
|
16
18
|
]
|
scythe/auth/base.py
CHANGED
|
@@ -80,6 +80,15 @@ class Authentication(ABC):
|
|
|
80
80
|
"""
|
|
81
81
|
return {}
|
|
82
82
|
|
|
83
|
+
def get_auth_cookies(self) -> Dict[str, str]:
|
|
84
|
+
"""
|
|
85
|
+
Get authentication cookies that should be set for API requests.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Dictionary mapping cookie name to cookie value.
|
|
89
|
+
"""
|
|
90
|
+
return {}
|
|
91
|
+
|
|
83
92
|
def store_auth_data(self, key: str, value: Any) -> None:
|
|
84
93
|
"""
|
|
85
94
|
Store authentication-related data.
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import Dict, Optional, Any
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import requests # type: ignore
|
|
9
|
+
except Exception: # pragma: no cover - tests may run without requests installed
|
|
10
|
+
requests = None # type: ignore
|
|
11
|
+
from selenium.webdriver.remote.webdriver import WebDriver
|
|
12
|
+
|
|
13
|
+
from .base import Authentication, AuthenticationError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _extract_by_dot_path(data: Any, path: str) -> Optional[Any]:
|
|
17
|
+
"""
|
|
18
|
+
Extract a value from a nested dict/list structure using a simple dot path.
|
|
19
|
+
Supports numeric indices for lists, e.g., "data.items.0.token".
|
|
20
|
+
"""
|
|
21
|
+
if not path:
|
|
22
|
+
return None
|
|
23
|
+
parts = path.split(".")
|
|
24
|
+
current: Any = data
|
|
25
|
+
for part in parts:
|
|
26
|
+
if isinstance(current, dict):
|
|
27
|
+
if part in current:
|
|
28
|
+
current = current[part]
|
|
29
|
+
else:
|
|
30
|
+
return None
|
|
31
|
+
elif isinstance(current, list):
|
|
32
|
+
try:
|
|
33
|
+
idx = int(part)
|
|
34
|
+
except ValueError:
|
|
35
|
+
return None
|
|
36
|
+
if idx < 0 or idx >= len(current):
|
|
37
|
+
return None
|
|
38
|
+
current = current[idx]
|
|
39
|
+
else:
|
|
40
|
+
return None
|
|
41
|
+
return current
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class CookieJWTAuth(Authentication):
|
|
45
|
+
"""
|
|
46
|
+
Hybrid authentication where a JWT is acquired from an API login response and
|
|
47
|
+
then used as a cookie for subsequent requests. Useful when the target server
|
|
48
|
+
expects a cookie (e.g., "stellarbridge") instead of Authorization headers.
|
|
49
|
+
|
|
50
|
+
Behavior:
|
|
51
|
+
- In API mode: JourneyExecutor will call get_auth_cookies(); this class will
|
|
52
|
+
perform a POST to login_url (if token not cached), parse JSON, extract the
|
|
53
|
+
token via jwt_json_path, and return {cookie_name: token}.
|
|
54
|
+
- In UI mode: authenticate() will ensure the browser has the cookie set for
|
|
55
|
+
the target domain.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(self,
|
|
59
|
+
login_url: str,
|
|
60
|
+
username: Optional[str] = None,
|
|
61
|
+
password: Optional[str] = None,
|
|
62
|
+
username_field: str = "email",
|
|
63
|
+
password_field: str = "password",
|
|
64
|
+
extra_fields: Optional[Dict[str, Any]] = None,
|
|
65
|
+
jwt_json_path: str = "token",
|
|
66
|
+
cookie_name: str = "stellarbridge",
|
|
67
|
+
session: Optional[requests.Session] = None,
|
|
68
|
+
description: str = "Authenticate via API and set JWT cookie"):
|
|
69
|
+
super().__init__(
|
|
70
|
+
name="Cookie JWT Authentication",
|
|
71
|
+
description=description
|
|
72
|
+
)
|
|
73
|
+
self.login_url = login_url
|
|
74
|
+
self.username = username
|
|
75
|
+
self.password = password
|
|
76
|
+
self.username_field = username_field
|
|
77
|
+
self.password_field = password_field
|
|
78
|
+
self.extra_fields = extra_fields or {}
|
|
79
|
+
self.jwt_json_path = jwt_json_path
|
|
80
|
+
self.cookie_name = cookie_name
|
|
81
|
+
# Avoid importing requests in test environments; allow injected session
|
|
82
|
+
self._session = session or (requests.Session() if requests is not None else None)
|
|
83
|
+
self.token: Optional[str] = None
|
|
84
|
+
|
|
85
|
+
def _login_and_get_token(self) -> str:
|
|
86
|
+
payload: Dict[str, Any] = dict(self.extra_fields)
|
|
87
|
+
if self.username is not None:
|
|
88
|
+
payload[self.username_field] = self.username
|
|
89
|
+
if self.password is not None:
|
|
90
|
+
payload[self.password_field] = self.password
|
|
91
|
+
try:
|
|
92
|
+
resp = self._session.post(self.login_url, json=payload, timeout=15)
|
|
93
|
+
# try json; raise on non-2xx to surface errors
|
|
94
|
+
resp.raise_for_status()
|
|
95
|
+
data = resp.json()
|
|
96
|
+
except Exception as e:
|
|
97
|
+
raise AuthenticationError(f"Login request failed: {e}", self.name)
|
|
98
|
+
token = _extract_by_dot_path(data, self.jwt_json_path)
|
|
99
|
+
if not token or not isinstance(token, str):
|
|
100
|
+
raise AuthenticationError(
|
|
101
|
+
f"JWT not found at path '{self.jwt_json_path}' in login response",
|
|
102
|
+
self.name,
|
|
103
|
+
)
|
|
104
|
+
self.token = token
|
|
105
|
+
self.store_auth_data('jwt', token)
|
|
106
|
+
self.store_auth_data('login_time', time.time())
|
|
107
|
+
return token
|
|
108
|
+
|
|
109
|
+
def get_auth_cookies(self) -> Dict[str, str]:
|
|
110
|
+
"""
|
|
111
|
+
Return cookie mapping for API mode. Will perform login if token absent.
|
|
112
|
+
"""
|
|
113
|
+
if not self.token:
|
|
114
|
+
self._login_and_get_token()
|
|
115
|
+
if not self.token:
|
|
116
|
+
return {}
|
|
117
|
+
return {self.cookie_name: self.token}
|
|
118
|
+
|
|
119
|
+
def get_auth_headers(self) -> Dict[str, str]:
|
|
120
|
+
"""
|
|
121
|
+
For this hybrid approach, we typically do not use auth headers.
|
|
122
|
+
"""
|
|
123
|
+
return {}
|
|
124
|
+
|
|
125
|
+
def authenticate(self, driver: WebDriver, target_url: str) -> bool:
|
|
126
|
+
"""
|
|
127
|
+
UI path: ensure the cookie exists on the browser for the target domain.
|
|
128
|
+
Will perform the API login if token not yet acquired.
|
|
129
|
+
"""
|
|
130
|
+
try:
|
|
131
|
+
if not self.token:
|
|
132
|
+
self._login_and_get_token()
|
|
133
|
+
if not self.token:
|
|
134
|
+
return False
|
|
135
|
+
# Navigate to the target domain base so cookie domain matches
|
|
136
|
+
parsed = urlparse(target_url)
|
|
137
|
+
base = f"{parsed.scheme}://{parsed.netloc}"
|
|
138
|
+
try:
|
|
139
|
+
driver.get(base)
|
|
140
|
+
except Exception:
|
|
141
|
+
pass
|
|
142
|
+
cookie_dict = {
|
|
143
|
+
'name': self.cookie_name,
|
|
144
|
+
'value': self.token,
|
|
145
|
+
'path': '/',
|
|
146
|
+
}
|
|
147
|
+
# If domain available, set explicitly to be safe
|
|
148
|
+
if parsed.netloc:
|
|
149
|
+
cookie_dict['domain'] = parsed.hostname or parsed.netloc
|
|
150
|
+
driver.add_cookie(cookie_dict)
|
|
151
|
+
self.authenticated = True
|
|
152
|
+
return True
|
|
153
|
+
except Exception as e:
|
|
154
|
+
raise AuthenticationError(f"Cookie auth failed: {e}", self.name)
|
|
155
|
+
|
|
156
|
+
def is_authenticated(self, driver: WebDriver) -> bool:
|
|
157
|
+
return self.authenticated and self.token is not None
|
|
158
|
+
|
|
159
|
+
def logout(self, driver: WebDriver) -> bool:
|
|
160
|
+
try:
|
|
161
|
+
self.token = None
|
|
162
|
+
self.authenticated = False
|
|
163
|
+
self.clear_auth_data()
|
|
164
|
+
# Best-effort cookie removal
|
|
165
|
+
try:
|
|
166
|
+
driver.delete_cookie(self.cookie_name)
|
|
167
|
+
except Exception:
|
|
168
|
+
pass
|
|
169
|
+
super().logout(driver)
|
|
170
|
+
return True
|
|
171
|
+
except Exception:
|
|
172
|
+
return False
|
scythe/core/headers.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
+
import requests
|
|
3
4
|
from typing import Optional, Dict, Any
|
|
4
5
|
from selenium.webdriver.remote.webdriver import WebDriver
|
|
5
6
|
from selenium.webdriver.chrome.options import Options
|
|
@@ -33,6 +34,157 @@ class HeaderExtractor:
|
|
|
33
34
|
chrome_options.add_argument("--log-level=0")
|
|
34
35
|
chrome_options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
|
|
35
36
|
|
|
37
|
+
def banner_grab(self, url: str, timeout: int = 10, method: str = "HEAD") -> Optional[str]:
|
|
38
|
+
"""
|
|
39
|
+
Perform a simple HTTP request to extract the X-SCYTHE-TARGET-VERSION header.
|
|
40
|
+
|
|
41
|
+
This is a more reliable alternative to Selenium's performance logging
|
|
42
|
+
for cases where you just need to grab headers.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
url: URL to make the request to
|
|
46
|
+
timeout: Request timeout in seconds
|
|
47
|
+
method: HTTP method to use ("HEAD" or "GET")
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
Version string if header found, None otherwise
|
|
51
|
+
"""
|
|
52
|
+
try:
|
|
53
|
+
self.logger.debug(f"Making {method} request to {url} for header extraction")
|
|
54
|
+
|
|
55
|
+
# Use HEAD by default for efficiency, fallback to GET if needed
|
|
56
|
+
if method.upper() == "HEAD":
|
|
57
|
+
response = requests.head(url, timeout=timeout, allow_redirects=True)
|
|
58
|
+
else:
|
|
59
|
+
response = requests.get(url, timeout=timeout, allow_redirects=True)
|
|
60
|
+
|
|
61
|
+
# Check if request was successful
|
|
62
|
+
response.raise_for_status()
|
|
63
|
+
|
|
64
|
+
# Look for the version header (case-insensitive)
|
|
65
|
+
version = self._find_version_header(dict(response.headers))
|
|
66
|
+
if version:
|
|
67
|
+
self.logger.debug(f"Found target version '{version}' via {method} request to {url}")
|
|
68
|
+
return version
|
|
69
|
+
else:
|
|
70
|
+
self.logger.debug(f"No X-SCYTHE-TARGET-VERSION header found in response from {url}")
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
except requests.exceptions.RequestException as e:
|
|
74
|
+
self.logger.warning(f"Failed to make {method} request to {url}: {e}")
|
|
75
|
+
return None
|
|
76
|
+
except Exception as e:
|
|
77
|
+
self.logger.warning(f"Unexpected error during banner grab: {e}")
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
def get_all_headers_via_request(self, url: str, timeout: int = 10, method: str = "HEAD") -> Dict[str, str]:
|
|
81
|
+
"""
|
|
82
|
+
Get all headers from a simple HTTP request.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
url: URL to make the request to
|
|
86
|
+
timeout: Request timeout in seconds
|
|
87
|
+
method: HTTP method to use ("HEAD" or "GET")
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
Dictionary of all response headers
|
|
91
|
+
"""
|
|
92
|
+
try:
|
|
93
|
+
self.logger.debug(f"Making {method} request to {url} for all headers")
|
|
94
|
+
|
|
95
|
+
if method.upper() == "HEAD":
|
|
96
|
+
response = requests.head(url, timeout=timeout, allow_redirects=True)
|
|
97
|
+
else:
|
|
98
|
+
response = requests.get(url, timeout=timeout, allow_redirects=True)
|
|
99
|
+
|
|
100
|
+
response.raise_for_status()
|
|
101
|
+
|
|
102
|
+
# Convert headers to regular dict with string values
|
|
103
|
+
return {k: str(v) for k, v in response.headers.items()}
|
|
104
|
+
|
|
105
|
+
except requests.exceptions.RequestException as e:
|
|
106
|
+
self.logger.warning(f"Failed to get headers from {url}: {e}")
|
|
107
|
+
return {}
|
|
108
|
+
except Exception as e:
|
|
109
|
+
self.logger.warning(f"Unexpected error getting headers: {e}")
|
|
110
|
+
return {}
|
|
111
|
+
|
|
112
|
+
def debug_headers(self, url: str, timeout: int = 10) -> None:
|
|
113
|
+
"""
|
|
114
|
+
Debug method to print all headers received from a URL.
|
|
115
|
+
|
|
116
|
+
This is useful for troubleshooting when headers aren't being detected properly.
|
|
117
|
+
It will show you exactly what headers the server is sending.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
url: URL to make the request to
|
|
121
|
+
timeout: Request timeout in seconds
|
|
122
|
+
"""
|
|
123
|
+
print(f"\n{'='*60}")
|
|
124
|
+
print(f"DEBUG: Header dump for {url}")
|
|
125
|
+
print(f"{'='*60}")
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
# Try HEAD request first
|
|
129
|
+
print("\n--- HEAD Request ---")
|
|
130
|
+
response = requests.head(url, timeout=timeout, allow_redirects=True)
|
|
131
|
+
print(f"Status Code: {response.status_code}")
|
|
132
|
+
print(f"Headers ({len(response.headers)} total):")
|
|
133
|
+
|
|
134
|
+
for name, value in response.headers.items():
|
|
135
|
+
print(f" {name}: {value}")
|
|
136
|
+
if "scythe" in name.lower() or "version" in name.lower():
|
|
137
|
+
print(" *** POTENTIAL VERSION HEADER ***")
|
|
138
|
+
|
|
139
|
+
# Try GET request
|
|
140
|
+
print("\n--- GET Request ---")
|
|
141
|
+
response = requests.get(url, timeout=timeout, allow_redirects=True)
|
|
142
|
+
print(f"Status Code: {response.status_code}")
|
|
143
|
+
print(f"Headers ({len(response.headers)} total):")
|
|
144
|
+
|
|
145
|
+
for name, value in response.headers.items():
|
|
146
|
+
print(f" {name}: {value}")
|
|
147
|
+
if "scythe" in name.lower() or "version" in name.lower():
|
|
148
|
+
print(" *** POTENTIAL VERSION HEADER ***")
|
|
149
|
+
|
|
150
|
+
# Check specifically for the target header
|
|
151
|
+
version = self._find_version_header(dict(response.headers))
|
|
152
|
+
print(f"\nTarget version extraction result: {version}")
|
|
153
|
+
|
|
154
|
+
except Exception as e:
|
|
155
|
+
print(f"ERROR: Failed to debug headers: {e}")
|
|
156
|
+
|
|
157
|
+
print(f"{'='*60}\n")
|
|
158
|
+
|
|
159
|
+
def extract_target_version_hybrid(self, driver: WebDriver, target_url: Optional[str] = None) -> Optional[str]:
|
|
160
|
+
"""
|
|
161
|
+
Hybrid approach: Try banner grab first, then fall back to Selenium performance logs.
|
|
162
|
+
|
|
163
|
+
This method attempts to get the version header using a simple HTTP request first,
|
|
164
|
+
which is more reliable than Selenium's performance logging. If that fails or no
|
|
165
|
+
target_url is provided, it falls back to the Selenium-based extraction.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
driver: WebDriver instance
|
|
169
|
+
target_url: URL to check (required for banner grab method)
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
Version string if header found, None otherwise
|
|
173
|
+
"""
|
|
174
|
+
# Try banner grab first if we have a target URL
|
|
175
|
+
if target_url:
|
|
176
|
+
self.logger.debug("Attempting banner grab method first")
|
|
177
|
+
version = self.banner_grab(target_url)
|
|
178
|
+
if version:
|
|
179
|
+
self.logger.debug(f"Successfully extracted version '{version}' via banner grab")
|
|
180
|
+
return version
|
|
181
|
+
else:
|
|
182
|
+
self.logger.debug("Banner grab failed, falling back to Selenium performance logs")
|
|
183
|
+
|
|
184
|
+
# Fall back to Selenium performance logs
|
|
185
|
+
self.logger.debug("Using Selenium performance logs method")
|
|
186
|
+
return self.extract_target_version(driver, target_url)
|
|
187
|
+
|
|
36
188
|
def extract_target_version(self, driver: WebDriver, target_url: Optional[str] = None) -> Optional[str]:
|
|
37
189
|
"""
|
|
38
190
|
Extract the X-SCYTHE-TARGET-VERSION header from the most recent HTTP response.
|
scythe/journeys/__init__.py
CHANGED
|
@@ -7,7 +7,7 @@ or other custom actions like navigation, form filling, etc.
|
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from .base import Journey, Step, Action
|
|
10
|
-
from .actions import NavigateAction, ClickAction, FillFormAction, WaitAction, TTPAction
|
|
10
|
+
from .actions import NavigateAction, ClickAction, FillFormAction, WaitAction, TTPAction, ApiRequestAction
|
|
11
11
|
from .executor import JourneyExecutor
|
|
12
12
|
|
|
13
13
|
__all__ = [
|
|
@@ -19,5 +19,6 @@ __all__ = [
|
|
|
19
19
|
'FillFormAction',
|
|
20
20
|
'WaitAction',
|
|
21
21
|
'TTPAction',
|
|
22
|
+
'ApiRequestAction',
|
|
22
23
|
'JourneyExecutor'
|
|
23
24
|
]
|
scythe/journeys/actions.py
CHANGED
|
@@ -5,6 +5,7 @@ from selenium.webdriver.common.by import By
|
|
|
5
5
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
6
6
|
from selenium.webdriver.support import expected_conditions as EC
|
|
7
7
|
from selenium.common.exceptions import TimeoutException, NoSuchElementException
|
|
8
|
+
import requests
|
|
8
9
|
|
|
9
10
|
from .base import Action
|
|
10
11
|
from ..core.ttp import TTP
|
|
@@ -643,4 +644,126 @@ class AssertAction(Action):
|
|
|
643
644
|
by_method = selector_map.get(self.selector_type)
|
|
644
645
|
if by_method is None:
|
|
645
646
|
raise ValueError(f"Unsupported selector type: {self.selector_type}")
|
|
646
|
-
return by_method
|
|
647
|
+
return by_method
|
|
648
|
+
|
|
649
|
+
class ApiRequestAction(Action):
|
|
650
|
+
"""Action to perform a REST API request in Journey API mode.
|
|
651
|
+
|
|
652
|
+
This action ignores the WebDriver and uses a requests.Session provided
|
|
653
|
+
in the journey context under the key 'requests_session'. It merges any
|
|
654
|
+
'auth_headers' from the context into the request headers.
|
|
655
|
+
Optionally, it can validate and parse the JSON response using a Pydantic
|
|
656
|
+
model class when provided.
|
|
657
|
+
"""
|
|
658
|
+
def __init__(self,
|
|
659
|
+
method: str,
|
|
660
|
+
url: str,
|
|
661
|
+
params: Optional[Dict[str, Any]] = None,
|
|
662
|
+
body_json: Optional[Dict[str, Any]] = None,
|
|
663
|
+
data: Optional[Dict[str, Any]] = None,
|
|
664
|
+
headers: Optional[Dict[str, str]] = None,
|
|
665
|
+
expected_status: Optional[int] = 200,
|
|
666
|
+
timeout: float = 10.0,
|
|
667
|
+
name: Optional[str] = None,
|
|
668
|
+
description: Optional[str] = None,
|
|
669
|
+
expected_result: bool = True,
|
|
670
|
+
response_model: Optional[Any] = None,
|
|
671
|
+
response_model_context_key: Optional[str] = None,
|
|
672
|
+
fail_on_validation_error: bool = False):
|
|
673
|
+
self.method = method.upper()
|
|
674
|
+
self.url = url
|
|
675
|
+
self.params = params or {}
|
|
676
|
+
self.body_json = body_json
|
|
677
|
+
self.data = data
|
|
678
|
+
self.headers = headers or {}
|
|
679
|
+
self.expected_status = expected_status
|
|
680
|
+
self.timeout = timeout
|
|
681
|
+
self.response_model = response_model
|
|
682
|
+
self.response_model_context_key = response_model_context_key
|
|
683
|
+
self.fail_on_validation_error = fail_on_validation_error
|
|
684
|
+
name = name or f"API {self.method} {url}"
|
|
685
|
+
description = description or f"Perform {self.method} request to {url}"
|
|
686
|
+
super().__init__(name, description, expected_result)
|
|
687
|
+
|
|
688
|
+
def execute(self, driver: WebDriver, context: Dict[str, Any]) -> bool:
|
|
689
|
+
# Resolve session
|
|
690
|
+
session = context.get('requests_session')
|
|
691
|
+
if session is None:
|
|
692
|
+
session = requests.Session()
|
|
693
|
+
context['requests_session'] = session
|
|
694
|
+
|
|
695
|
+
# Build headers: auth headers from context + action headers (action overrides)
|
|
696
|
+
final_headers = {}
|
|
697
|
+
auth_headers = context.get('auth_headers', {}) or {}
|
|
698
|
+
if auth_headers:
|
|
699
|
+
final_headers.update(auth_headers)
|
|
700
|
+
if self.headers:
|
|
701
|
+
final_headers.update(self.headers)
|
|
702
|
+
|
|
703
|
+
# Resolve URL: absolute or join with target_url from context
|
|
704
|
+
from urllib.parse import urljoin
|
|
705
|
+
base_url = context.get('target_url') or ''
|
|
706
|
+
resolved_url = self.url if self.url.lower().startswith('http') else urljoin(base_url, self.url)
|
|
707
|
+
|
|
708
|
+
try:
|
|
709
|
+
response = session.request(
|
|
710
|
+
self.method,
|
|
711
|
+
resolved_url,
|
|
712
|
+
params=self.params or None,
|
|
713
|
+
json=self.body_json,
|
|
714
|
+
data=self.data,
|
|
715
|
+
headers=final_headers or None,
|
|
716
|
+
timeout=self.timeout,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
# Store details
|
|
720
|
+
self.store_result('url', resolved_url)
|
|
721
|
+
self.store_result('request_headers', final_headers)
|
|
722
|
+
self.store_result('status_code', getattr(response, 'status_code', None))
|
|
723
|
+
response_headers = dict(getattr(response, 'headers', {}) or {})
|
|
724
|
+
self.store_result('response_headers', response_headers)
|
|
725
|
+
# Publish to context for downstream version extraction
|
|
726
|
+
context['last_response_headers'] = response_headers
|
|
727
|
+
context['last_response_url'] = resolved_url
|
|
728
|
+
# Try JSON, fallback to text
|
|
729
|
+
body = None
|
|
730
|
+
parsed_model = None
|
|
731
|
+
validation_error = None
|
|
732
|
+
try:
|
|
733
|
+
body = response.json()
|
|
734
|
+
self.store_result('response_json', body)
|
|
735
|
+
# If a response_model is provided, attempt validation/parsing
|
|
736
|
+
if self.response_model is not None:
|
|
737
|
+
try:
|
|
738
|
+
# Pydantic v2 preferred: model_validate
|
|
739
|
+
if hasattr(self.response_model, 'model_validate'):
|
|
740
|
+
parsed_model = self.response_model.model_validate(body)
|
|
741
|
+
else:
|
|
742
|
+
# Pydantic v1 fallback
|
|
743
|
+
parsed_model = self.response_model.parse_obj(body)
|
|
744
|
+
self.store_result('response_model_instance', parsed_model)
|
|
745
|
+
# Save into context for downstream actions
|
|
746
|
+
key = self.response_model_context_key or 'last_response_model'
|
|
747
|
+
context[key] = parsed_model
|
|
748
|
+
except Exception as ve:
|
|
749
|
+
validation_error = str(ve)
|
|
750
|
+
self.store_result('response_validation_error', validation_error)
|
|
751
|
+
except Exception:
|
|
752
|
+
text = getattr(response, 'text', '')
|
|
753
|
+
# Limit stored text to keep logs light
|
|
754
|
+
self.store_result('response_text', text if text is None or len(text) <= 2000 else text[:2000])
|
|
755
|
+
|
|
756
|
+
# Determine success (status-based by default)
|
|
757
|
+
if self.expected_status is not None:
|
|
758
|
+
http_ok = (getattr(response, 'status_code', None) == self.expected_status)
|
|
759
|
+
else:
|
|
760
|
+
http_ok = bool(getattr(response, 'ok', False))
|
|
761
|
+
|
|
762
|
+
# Optionally fail on validation error
|
|
763
|
+
if self.response_model is not None and self.fail_on_validation_error and self.get_result('response_validation_error'):
|
|
764
|
+
return False if http_ok else False
|
|
765
|
+
|
|
766
|
+
return http_ok
|
|
767
|
+
except Exception as e:
|
|
768
|
+
self.store_result('error', str(e))
|
|
769
|
+
return False
|
scythe/journeys/base.py
CHANGED
|
@@ -311,12 +311,45 @@ class Journey:
|
|
|
311
311
|
if self.requires_authentication():
|
|
312
312
|
auth_name = self.authentication.name if self.authentication else "Unknown"
|
|
313
313
|
logger.info(f"Authentication required: {auth_name}")
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
314
|
+
if driver is None:
|
|
315
|
+
# API mode: use header-based authentication if available
|
|
316
|
+
headers = {}
|
|
317
|
+
try:
|
|
318
|
+
if self.authentication and hasattr(self.authentication, 'get_auth_headers'):
|
|
319
|
+
headers = self.authentication.get_auth_headers() or {}
|
|
320
|
+
except Exception as e:
|
|
321
|
+
logger.error(f"Failed to get authentication headers: {e}")
|
|
322
|
+
headers = {}
|
|
323
|
+
cookies = {}
|
|
324
|
+
if headers:
|
|
325
|
+
# Merge into existing context headers
|
|
326
|
+
existing = self.get_context('auth_headers', {})
|
|
327
|
+
merged = {**existing, **headers}
|
|
328
|
+
self.set_context('auth_headers', merged)
|
|
329
|
+
logger.info("Authentication headers prepared for API mode")
|
|
330
|
+
# Try to merge cookies as well (hybrid auth)
|
|
331
|
+
try:
|
|
332
|
+
if self.authentication and hasattr(self.authentication, 'get_auth_cookies'):
|
|
333
|
+
cookies = self.authentication.get_auth_cookies() or {}
|
|
334
|
+
except Exception as e:
|
|
335
|
+
logger.error(f"Failed to get authentication cookies: {e}")
|
|
336
|
+
cookies = {}
|
|
337
|
+
if cookies:
|
|
338
|
+
existing_cookies = self.get_context('auth_cookies', {})
|
|
339
|
+
merged_cookies = {**existing_cookies, **cookies}
|
|
340
|
+
self.set_context('auth_cookies', merged_cookies)
|
|
341
|
+
logger.info("Authentication cookies prepared for API mode")
|
|
342
|
+
if not headers and not cookies:
|
|
343
|
+
logger.error("Authentication required but no headers/cookies available in API mode")
|
|
344
|
+
results['errors'].append("Authentication failed (no API auth data)")
|
|
345
|
+
return results
|
|
346
|
+
else:
|
|
347
|
+
auth_success = self.authenticate(driver, target_url)
|
|
348
|
+
if not auth_success:
|
|
349
|
+
logger.error("Authentication failed - aborting journey")
|
|
350
|
+
results['errors'].append("Authentication failed")
|
|
351
|
+
return results
|
|
352
|
+
logger.info("Authentication successful")
|
|
320
353
|
|
|
321
354
|
# Execute each step
|
|
322
355
|
for i, step in enumerate(self.steps, 1):
|
|
@@ -342,7 +375,7 @@ class Journey:
|
|
|
342
375
|
results['actions_failed'] += 1
|
|
343
376
|
|
|
344
377
|
# Extract target version header after step execution
|
|
345
|
-
target_version = header_extractor.
|
|
378
|
+
target_version = header_extractor.extract_target_version_hybrid(driver, target_url)
|
|
346
379
|
if target_version:
|
|
347
380
|
results['target_versions'].append(target_version)
|
|
348
381
|
logger.info(f"Target version detected: {target_version}")
|
scythe/journeys/executor.py
CHANGED
|
@@ -3,6 +3,7 @@ import logging
|
|
|
3
3
|
from selenium import webdriver
|
|
4
4
|
from selenium.webdriver.chrome.options import Options
|
|
5
5
|
from typing import Optional, Dict, Any, List
|
|
6
|
+
import requests
|
|
6
7
|
from ..behaviors.base import Behavior
|
|
7
8
|
from .base import Journey
|
|
8
9
|
from ..core.headers import HeaderExtractor
|
|
@@ -24,6 +25,10 @@ class JourneyExecutor:
|
|
|
24
25
|
|
|
25
26
|
Similar to TTPExecutor but designed for complex multi-step scenarios
|
|
26
27
|
involving journeys composed of steps and actions.
|
|
28
|
+
|
|
29
|
+
Supports two interaction modes:
|
|
30
|
+
- UI: browser-driven via Selenium (default, backward-compatible)
|
|
31
|
+
- API: REST-driven via requests without starting a browser
|
|
27
32
|
"""
|
|
28
33
|
|
|
29
34
|
def __init__(self,
|
|
@@ -31,7 +36,8 @@ class JourneyExecutor:
|
|
|
31
36
|
target_url: str,
|
|
32
37
|
headless: bool = True,
|
|
33
38
|
behavior: Optional[Behavior] = None,
|
|
34
|
-
driver_options: Optional[Dict[str, Any]] = None
|
|
39
|
+
driver_options: Optional[Dict[str, Any]] = None,
|
|
40
|
+
mode: str = "UI"):
|
|
35
41
|
"""
|
|
36
42
|
Initialize the Journey executor.
|
|
37
43
|
|
|
@@ -45,6 +51,7 @@ class JourneyExecutor:
|
|
|
45
51
|
self.journey = journey
|
|
46
52
|
self.target_url = target_url
|
|
47
53
|
self.behavior = behavior
|
|
54
|
+
self.mode = (mode or "UI").upper()
|
|
48
55
|
self.logger = logging.getLogger(f"Journey.{self.journey.name}")
|
|
49
56
|
|
|
50
57
|
# Setup Chrome options
|
|
@@ -108,29 +115,62 @@ class JourneyExecutor:
|
|
|
108
115
|
self.logger.info(f"Using behavior: {self.behavior.name}")
|
|
109
116
|
self.logger.info(f"Behavior description: {self.behavior.description}")
|
|
110
117
|
|
|
111
|
-
self._setup_driver()
|
|
112
|
-
|
|
113
118
|
try:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
if self.mode == 'API':
|
|
120
|
+
# API mode: no WebDriver, prepare requests session and context
|
|
121
|
+
session = requests.Session()
|
|
122
|
+
auth_headers = {}
|
|
123
|
+
auth_cookies = {}
|
|
124
|
+
if getattr(self.journey, 'authentication', None):
|
|
125
|
+
# Try to obtain headers/cookies directly (no browser flow)
|
|
126
|
+
try:
|
|
127
|
+
auth_headers = self.journey.authentication.get_auth_headers() or {}
|
|
128
|
+
except Exception as e:
|
|
129
|
+
self.logger.warning(f"Failed to get auth headers from authentication: {e}")
|
|
130
|
+
try:
|
|
131
|
+
if hasattr(self.journey.authentication, 'get_auth_cookies'):
|
|
132
|
+
auth_cookies = self.journey.authentication.get_auth_cookies() or {}
|
|
133
|
+
except Exception as e:
|
|
134
|
+
self.logger.warning(f"Failed to get auth cookies from authentication: {e}")
|
|
135
|
+
if auth_headers:
|
|
136
|
+
session.headers.update(auth_headers)
|
|
137
|
+
if auth_cookies:
|
|
138
|
+
for ck, cv in auth_cookies.items():
|
|
139
|
+
try:
|
|
140
|
+
session.cookies.set(ck, cv)
|
|
141
|
+
except Exception:
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
# Seed journey context for API actions
|
|
145
|
+
self.journey.set_context('mode', 'API')
|
|
146
|
+
self.journey.set_context('requests_session', session)
|
|
147
|
+
self.journey.set_context('auth_headers', auth_headers)
|
|
148
|
+
self.journey.set_context('auth_cookies', auth_cookies)
|
|
149
|
+
|
|
150
|
+
# Execute journey with a None driver (API actions ignore driver)
|
|
151
|
+
self.execution_results = self.journey.execute(None, self.target_url)
|
|
121
152
|
else:
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
153
|
+
# UI mode (default)
|
|
154
|
+
self._setup_driver()
|
|
155
|
+
|
|
156
|
+
# Pre-execution behavior setup
|
|
157
|
+
if self.behavior and self.driver:
|
|
158
|
+
self.behavior.pre_execution(self.driver, self.target_url)
|
|
159
|
+
|
|
160
|
+
# Execute the journey
|
|
161
|
+
if self.driver:
|
|
162
|
+
self.execution_results = self.journey.execute(self.driver, self.target_url)
|
|
163
|
+
else:
|
|
164
|
+
raise RuntimeError("WebDriver not initialized")
|
|
165
|
+
|
|
166
|
+
# Apply behavior timing between steps if configured
|
|
167
|
+
if self.behavior:
|
|
168
|
+
self._apply_behavior_to_journey()
|
|
169
|
+
|
|
170
|
+
# Post-execution behavior cleanup
|
|
171
|
+
if self.behavior and self.driver:
|
|
172
|
+
behavior_results = self._convert_results_for_behavior()
|
|
173
|
+
self.behavior.post_execution(self.driver, behavior_results)
|
|
134
174
|
|
|
135
175
|
except KeyboardInterrupt:
|
|
136
176
|
self.logger.info("Journey interrupted by user.")
|
|
@@ -143,6 +183,7 @@ class JourneyExecutor:
|
|
|
143
183
|
self.execution_results = self._create_error_results(str(e))
|
|
144
184
|
|
|
145
185
|
finally:
|
|
186
|
+
# Cleanup and print summary (driver quit only if initialized)
|
|
146
187
|
self._cleanup()
|
|
147
188
|
|
|
148
189
|
return self.execution_results
|
scythe/ttps/web/uuid_guessing.py
CHANGED
|
@@ -5,6 +5,7 @@ from ...payloads.generators import PayloadGenerator
|
|
|
5
5
|
import requests
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
8
|
+
|
|
8
9
|
class GuessUUIDInURL(TTP):
|
|
9
10
|
def __init__(self,
|
|
10
11
|
target_url: str,
|
|
@@ -18,10 +19,10 @@ class GuessUUIDInURL(TTP):
|
|
|
18
19
|
description="simulate bruteforcing UUID's in the URL path",
|
|
19
20
|
expected_result=expected_result,
|
|
20
21
|
authentication=authentication)
|
|
21
|
-
|
|
22
|
+
|
|
22
23
|
self.target_url = target_url
|
|
23
24
|
self.uri_root_path = uri_root_path
|
|
24
|
-
self.payload_generator = payload_generator
|
|
25
|
+
self.payload_generator = payload_generator
|
|
25
26
|
|
|
26
27
|
def get_payloads(self):
|
|
27
28
|
yield from self.payload_generator()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scythe-ttp
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: An extensible framework for emulating attacker TTPs with Selenium.
|
|
5
5
|
Home-page: https://github.com/EpykLab/scythe
|
|
6
6
|
Author: EpykLab
|
|
@@ -23,6 +23,8 @@ Requires-Dist: h11==0.16.0
|
|
|
23
23
|
Requires-Dist: idna==3.10
|
|
24
24
|
Requires-Dist: outcome==1.3.0.post0
|
|
25
25
|
Requires-Dist: PySocks==1.7.1
|
|
26
|
+
Requires-Dist: pydantic==2.7.1
|
|
27
|
+
Requires-Dist: pydantic-core==2.18.2
|
|
26
28
|
Requires-Dist: requests==2.32.4
|
|
27
29
|
Requires-Dist: selenium==4.34.0
|
|
28
30
|
Requires-Dist: setuptools==80.9.0
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
scythe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
scythe/auth/__init__.py,sha256=
|
|
3
|
-
scythe/auth/base.py,sha256=
|
|
2
|
+
scythe/auth/__init__.py,sha256=InEANqWEIAULFyzH9IyxWDPs_gJd3m_JYmzoaBk_37M,420
|
|
3
|
+
scythe/auth/base.py,sha256=DllKaPGj0MRyRh4PQgQ2TUFgeAXjgXOT2h6zUz2ZAag,3807
|
|
4
4
|
scythe/auth/basic.py,sha256=H4IG9-Y7wFe7ZQCNHmmqhre-Pp9CnBxlT23h2uvOPWo,14354
|
|
5
5
|
scythe/auth/bearer.py,sha256=ngOL-sS6FcfB8XAKMR-CZbpqyySu2MaKxUl10SyBmmI,12687
|
|
6
|
+
scythe/auth/cookie_jwt.py,sha256=z5Q-c594_m-dmh2Rv_4Xfeu0fXSQdlZ12Q-emtyj63g,6337
|
|
6
7
|
scythe/behaviors/__init__.py,sha256=w-WRBGRgna5a1N8oHP2aXSQnkQUHyOXiujpwEVf_ZyM,291
|
|
7
8
|
scythe/behaviors/base.py,sha256=INvIYKVIWzEi5w_4njOwKZ3X9IvySvqiMJnYX7_2Lns,3955
|
|
8
9
|
scythe/behaviors/default.py,sha256=MDx4N-KwC23pPLGu1-ZIkGiTRNUG3Lxjbvo7SJ3UwMc,2117
|
|
@@ -11,12 +12,12 @@ scythe/behaviors/machine.py,sha256=NDMUq3mDhpCvujzAFxhn2eSVq78-J-LSBhIcvHkzKXo,4
|
|
|
11
12
|
scythe/behaviors/stealth.py,sha256=xv7MrPQgRCdCUJyYTcXV2aasWZoAw8rAQWg-AuQVb7U,15278
|
|
12
13
|
scythe/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
14
|
scythe/core/executor.py,sha256=x1w2nByVu2G70sh7t0kOh6urlrTm_r_pbk0S7v1Ov28,9736
|
|
14
|
-
scythe/core/headers.py,sha256=
|
|
15
|
+
scythe/core/headers.py,sha256=lHlxAwV4xOoR9A0rHDM5k48HmLp1y03uOkXXq68vcAU,13867
|
|
15
16
|
scythe/core/ttp.py,sha256=Xw9GgptYsjZ-pMLdyPv64bhiwGKobrXHdF32pjIY7OU,3102
|
|
16
|
-
scythe/journeys/__init__.py,sha256
|
|
17
|
-
scythe/journeys/actions.py,sha256=
|
|
18
|
-
scythe/journeys/base.py,sha256=
|
|
19
|
-
scythe/journeys/executor.py,sha256=
|
|
17
|
+
scythe/journeys/__init__.py,sha256=Odi8NhRg7Hefmo1EJj1guakrCSPhsuus4i-_62uUUjs,654
|
|
18
|
+
scythe/journeys/actions.py,sha256=seUZFlqda5QbaokoZF2CDgjKnpuL_PasQyK9eLPXwwY,31013
|
|
19
|
+
scythe/journeys/base.py,sha256=K8pzIbNqI_KX5jjsr6xQIxDWY3fD4dBmR5CxvR-9Cp4,17508
|
|
20
|
+
scythe/journeys/executor.py,sha256=Spv4JtHUWgJ8nZX__gSkC-cQthWvj5wrfR9Yqbr2XTA,22172
|
|
20
21
|
scythe/orchestrators/__init__.py,sha256=_vemcXjKbB1jI0F2dPA0F1zNsyUekjcXImLDUDhWDN0,560
|
|
21
22
|
scythe/orchestrators/base.py,sha256=YOZV0ewlzJ49H08P_LKnimutUms8NnDrQprFpSKhOeM,13595
|
|
22
23
|
scythe/orchestrators/batch.py,sha256=FpK501kk-earJzz6v7dcuw2y708rTvt_IMH_5qjKdrc,26635
|
|
@@ -28,9 +29,9 @@ scythe/ttps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
28
29
|
scythe/ttps/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
30
|
scythe/ttps/web/login_bruteforce.py,sha256=D4G8zB_nU9LD5w3Vv2ABTuOl4XTeg2BgZwYMObt4JJw,2488
|
|
30
31
|
scythe/ttps/web/sql_injection.py,sha256=aWk4DFePbtFDsieOOj03Ux-5OiykyOs2_d_3SvWMOVE,2910
|
|
31
|
-
scythe/ttps/web/uuid_guessing.py,sha256=
|
|
32
|
-
scythe_ttp-0.
|
|
33
|
-
scythe_ttp-0.
|
|
34
|
-
scythe_ttp-0.
|
|
35
|
-
scythe_ttp-0.
|
|
36
|
-
scythe_ttp-0.
|
|
32
|
+
scythe/ttps/web/uuid_guessing.py,sha256=JwNt_9HVynMWFPPU6UGJFcpxvMVDsvc_wAnJVtcYbps,1235
|
|
33
|
+
scythe_ttp-0.13.0.dist-info/licenses/LICENSE,sha256=B7iB4Fv6zDQolC7IgqNF8F4GEp_DLe2jrPPuR_MYMOM,1064
|
|
34
|
+
scythe_ttp-0.13.0.dist-info/METADATA,sha256=BWH_KWCnInk0y8PNmJPN2fxJ_7RavvhSppmoG8BtA3M,27809
|
|
35
|
+
scythe_ttp-0.13.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
36
|
+
scythe_ttp-0.13.0.dist-info/top_level.txt,sha256=BCKTrPuVvmLyhOR07C1ggOh6sU7g2LoVvwDMn46O55Y,7
|
|
37
|
+
scythe_ttp-0.13.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|