hellhound-spider 14.2__tar.gz → 14.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hellhound_spider-14.2 → hellhound_spider-14.3}/PKG-INFO +11 -3
- {hellhound_spider-14.2 → hellhound_spider-14.3}/README.md +10 -2
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/PKG-INFO +11 -3
- {hellhound_spider-14.2 → hellhound_spider-14.3}/man/spider.1 +18 -4
- {hellhound_spider-14.2 → hellhound_spider-14.3}/pyproject.toml +1 -1
- {hellhound_spider-14.2 → hellhound_spider-14.3}/spider.py +449 -8
- {hellhound_spider-14.2 → hellhound_spider-14.3}/LICENSE +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/SOURCES.txt +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/dependency_links.txt +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/entry_points.txt +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/requires.txt +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/top_level.txt +0 -0
- {hellhound_spider-14.2 → hellhound_spider-14.3}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hellhound-spider
|
|
3
|
-
Version: 14.
|
|
3
|
+
Version: 14.3
|
|
4
4
|
Summary: Autonomous web crawler and recon engine for security testing — SPA + Non-SPA crawling, header analysis, subdomain enumeration, and intelligence classification
|
|
5
5
|
Author-email: Sree Danush S <lazzer@gmail.com>
|
|
6
6
|
License-Expression: GPL-3.0-or-later
|
|
@@ -46,11 +46,14 @@ Dynamic: license-file
|
|
|
46
46
|
|
|
47
47
|
<p align="center">
|
|
48
48
|
<img src="https://img.shields.io/badge/python-3.10+-blue?style=flat-square&logo=python&logoColor=white"/>
|
|
49
|
-
<img src="https://img.shields.io/badge/version-14.
|
|
49
|
+
<img src="https://img.shields.io/badge/version-14.3-red?style=flat-square"/>
|
|
50
50
|
<img src="https://img.shields.io/badge/pypi-hellhound--spider-blue?style=flat-square&logo=pypi&logoColor=white"/>
|
|
51
51
|
<img src="https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey?style=flat-square"/>
|
|
52
52
|
<img src="https://img.shields.io/badge/license-GPL--3.0-blue?style=flat-square"/>
|
|
53
53
|
</p>
|
|
54
|
+
<p align="center">
|
|
55
|
+
<a href="https://devlist.app/listings/hellhound-spider"><img src="https://devlist.app/badges/listed-on-devlist-large.svg" alt="Listed on DevList" width="200"/></a>
|
|
56
|
+
</p>
|
|
54
57
|
|
|
55
58
|
---
|
|
56
59
|
|
|
@@ -174,7 +177,12 @@ spider https://target.com --ctf-flag "HELLCORP{},FLAG{}"
|
|
|
174
177
|
|
|
175
178
|
### Discovery Vectors
|
|
176
179
|
|
|
177
|
-
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting,
|
|
180
|
+
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting, JS SCA analysis, OAuth/SSO redirect detection and provider fingerprinting (Okta, Azure AD, Google, Auth0, and more), and CAPTCHA detection.
|
|
181
|
+
|
|
182
|
+
### Authentication
|
|
183
|
+
|
|
184
|
+
- `-l/--login URL -n/--login-user USER -z/--login-pass PASS` — automates a classic HTML username/password login form and carries the session through the rest of the scan. Bails out with a report instead of guessing if the target turns out to be OAuth/SSO-gated or CAPTCHA-protected.
|
|
185
|
+
- `-C/--cookie` — for everything `--login` can't handle (OAuth/SSO, MFA, anything scripted login isn't meant for): log in once by hand and hand Spider the session cookie.
|
|
178
186
|
|
|
179
187
|
### Parameter Mining
|
|
180
188
|
|
|
@@ -10,11 +10,14 @@
|
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<img src="https://img.shields.io/badge/python-3.10+-blue?style=flat-square&logo=python&logoColor=white"/>
|
|
13
|
-
<img src="https://img.shields.io/badge/version-14.
|
|
13
|
+
<img src="https://img.shields.io/badge/version-14.3-red?style=flat-square"/>
|
|
14
14
|
<img src="https://img.shields.io/badge/pypi-hellhound--spider-blue?style=flat-square&logo=pypi&logoColor=white"/>
|
|
15
15
|
<img src="https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey?style=flat-square"/>
|
|
16
16
|
<img src="https://img.shields.io/badge/license-GPL--3.0-blue?style=flat-square"/>
|
|
17
17
|
</p>
|
|
18
|
+
<p align="center">
|
|
19
|
+
<a href="https://devlist.app/listings/hellhound-spider"><img src="https://devlist.app/badges/listed-on-devlist-large.svg" alt="Listed on DevList" width="200"/></a>
|
|
20
|
+
</p>
|
|
18
21
|
|
|
19
22
|
---
|
|
20
23
|
|
|
@@ -138,7 +141,12 @@ spider https://target.com --ctf-flag "HELLCORP{},FLAG{}"
|
|
|
138
141
|
|
|
139
142
|
### Discovery Vectors
|
|
140
143
|
|
|
141
|
-
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting,
|
|
144
|
+
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting, JS SCA analysis, OAuth/SSO redirect detection and provider fingerprinting (Okta, Azure AD, Google, Auth0, and more), and CAPTCHA detection.
|
|
145
|
+
|
|
146
|
+
### Authentication
|
|
147
|
+
|
|
148
|
+
- `-l/--login URL -n/--login-user USER -z/--login-pass PASS` — automates a classic HTML username/password login form and carries the session through the rest of the scan. Bails out with a report instead of guessing if the target turns out to be OAuth/SSO-gated or CAPTCHA-protected.
|
|
149
|
+
- `-C/--cookie` — for everything `--login` can't handle (OAuth/SSO, MFA, anything scripted login isn't meant for): log in once by hand and hand Spider the session cookie.
|
|
142
150
|
|
|
143
151
|
### Parameter Mining
|
|
144
152
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hellhound-spider
|
|
3
|
-
Version: 14.
|
|
3
|
+
Version: 14.3
|
|
4
4
|
Summary: Autonomous web crawler and recon engine for security testing — SPA + Non-SPA crawling, header analysis, subdomain enumeration, and intelligence classification
|
|
5
5
|
Author-email: Sree Danush S <lazzer@gmail.com>
|
|
6
6
|
License-Expression: GPL-3.0-or-later
|
|
@@ -46,11 +46,14 @@ Dynamic: license-file
|
|
|
46
46
|
|
|
47
47
|
<p align="center">
|
|
48
48
|
<img src="https://img.shields.io/badge/python-3.10+-blue?style=flat-square&logo=python&logoColor=white"/>
|
|
49
|
-
<img src="https://img.shields.io/badge/version-14.
|
|
49
|
+
<img src="https://img.shields.io/badge/version-14.3-red?style=flat-square"/>
|
|
50
50
|
<img src="https://img.shields.io/badge/pypi-hellhound--spider-blue?style=flat-square&logo=pypi&logoColor=white"/>
|
|
51
51
|
<img src="https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey?style=flat-square"/>
|
|
52
52
|
<img src="https://img.shields.io/badge/license-GPL--3.0-blue?style=flat-square"/>
|
|
53
53
|
</p>
|
|
54
|
+
<p align="center">
|
|
55
|
+
<a href="https://devlist.app/listings/hellhound-spider"><img src="https://devlist.app/badges/listed-on-devlist-large.svg" alt="Listed on DevList" width="200"/></a>
|
|
56
|
+
</p>
|
|
54
57
|
|
|
55
58
|
---
|
|
56
59
|
|
|
@@ -174,7 +177,12 @@ spider https://target.com --ctf-flag "HELLCORP{},FLAG{}"
|
|
|
174
177
|
|
|
175
178
|
### Discovery Vectors
|
|
176
179
|
|
|
177
|
-
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting,
|
|
180
|
+
HTML crawl, live SPA XHR interception, parent-directory backtracking (probes ancestor directories the forward crawl never linked to), Intelligent Robots Analysis (Disallow/Allow mapping + comment mining), sitemap XML (with index recursion), `.well-known` (OIDC/JWKS), JSON path chaining, SPA hash routes, lazy-load attributes, CSP header hints, OpenAPI/Swagger specs, GraphQL introspection, crt.sh certificate transparency, Wayback Machine CDX API, security.txt (RFC 9116), HAR file import, HTML comment mining, response header analysis, TLS/DNS intel, WAF fingerprinting, JS SCA analysis, OAuth/SSO redirect detection and provider fingerprinting (Okta, Azure AD, Google, Auth0, and more), and CAPTCHA detection.
|
|
181
|
+
|
|
182
|
+
### Authentication
|
|
183
|
+
|
|
184
|
+
- `-l/--login URL -n/--login-user USER -z/--login-pass PASS` — automates a classic HTML username/password login form and carries the session through the rest of the scan. Bails out with a report instead of guessing if the target turns out to be OAuth/SSO-gated or CAPTCHA-protected.
|
|
185
|
+
- `-C/--cookie` — for everything `--login` can't handle (OAuth/SSO, MFA, anything scripted login isn't meant for): log in once by hand and hand Spider the session cookie.
|
|
178
186
|
|
|
179
187
|
### Parameter Mining
|
|
180
188
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
.TH SPIDER 1 "2026-11-
|
|
1
|
+
.TH SPIDER 1 "2026-11-06" "Hellhound Spider v14.3" "User Commands"
|
|
2
2
|
.SH NAME
|
|
3
3
|
spider \- SPA and non-SPA web crawler, endpoint discoverer, and attack-surface mapper
|
|
4
4
|
.SH SYNOPSIS
|
|
@@ -82,9 +82,9 @@ Show all discovery logs, including low-confidence and informational events.
|
|
|
82
82
|
.TP
|
|
83
83
|
.BR \-C ", " \-\-cookie " " \fICOOKIE\fR
|
|
84
84
|
Cookie string (e.g. \fB"session=42"\fR), dict-style string, or path to a cookie
|
|
85
|
-
file. Used to scan authenticated areas of the application.
|
|
86
|
-
|
|
87
|
-
|
|
85
|
+
file. Used to scan authenticated areas of the application. For an OAuth/SSO-gated
|
|
86
|
+
target, or any login flow \fB\-\-login\fR can't handle, log in manually via a
|
|
87
|
+
browser and supply the resulting session cookie here instead.
|
|
88
88
|
.TP
|
|
89
89
|
.BR \-a ", " \-\-auth " " \fIHEADER\fR
|
|
90
90
|
Authorization header value, e.g. \fB"Bearer eyJ..."\fR.
|
|
@@ -96,6 +96,20 @@ this is for server-level basic authentication, not standard web login forms.
|
|
|
96
96
|
.BR \-X ", " \-\-header " " \fI"NAME: VALUE"\fR
|
|
97
97
|
Custom HTTP header, repeatable. Pass custom headers formatted as \fB"Name: Value"\fR.
|
|
98
98
|
Useful for program-required tester identification or custom tokens.
|
|
99
|
+
.TP
|
|
100
|
+
.BR \-l ", " \-\-login " " \fIURL\fR
|
|
101
|
+
URL of a classic HTML username/password login form. Spider parses the form,
|
|
102
|
+
submits \fB\-n\fR/\fB\-z\fR (\fB\-\-login\-user\fR/\fB\-\-login\-pass\fR), and
|
|
103
|
+
carries the resulting session cookies through the rest of the scan. Aborts
|
|
104
|
+
with a clear report (no submission attempted) if the page redirects to a
|
|
105
|
+
third-party OAuth/SSO provider or serves a CAPTCHA \(em neither is something
|
|
106
|
+
Spider will try to bypass; use \fB\-\-cookie\fR instead for those.
|
|
107
|
+
.TP
|
|
108
|
+
.BR \-n ", " \-\-login\-user " " \fIUSERNAME\fR
|
|
109
|
+
Username/email to submit with \fB\-l\fR/\fB\-\-login\fR.
|
|
110
|
+
.TP
|
|
111
|
+
.BR \-z ", " \-\-login\-pass " " \fIPASSWORD\fR
|
|
112
|
+
Password to submit with \fB\-l\fR/\fB\-\-login\fR.
|
|
99
113
|
.SS "Output"
|
|
100
114
|
.TP
|
|
101
115
|
.BR \-o ", " \-\-out " " \fIFILE\fR
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hellhound-spider"
|
|
7
|
-
version = "14.
|
|
7
|
+
version = "14.3"
|
|
8
8
|
description = "Autonomous web crawler and recon engine for security testing — SPA + Non-SPA crawling, header analysis, subdomain enumeration, and intelligence classification"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -61,7 +61,7 @@ except Exception:
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
VERSION = "14.
|
|
64
|
+
VERSION = "14.3"
|
|
65
65
|
__author__ = "Sree Danush S (L4ZZ3RJ0D)"
|
|
66
66
|
__license__ = "GPLv3"
|
|
67
67
|
__credits__ = ["L4ZZ3RJ0D"]
|
|
@@ -837,7 +837,68 @@ def print_results(intel: dict, target: str, elapsed: float,
|
|
|
837
837
|
pill = f"{sev_col}{C.B} {sev} {C.RST}"
|
|
838
838
|
print(f" {pill} {C.W}{f.get('issue','')}{C.RST} {C.GR}{f.get('detail','')}{C.RST}")
|
|
839
839
|
|
|
840
|
-
|
|
840
|
+
|
|
841
|
+
# ── AUTH / OAUTH / SSO / CAPTCHA / LOGIN — "summary is what most
|
|
842
|
+
# hackers actually read" per explicit design intent, so all of this
|
|
843
|
+
# surfaces here, not just in the live feed. ────────────────────────
|
|
844
|
+
login_result = intel.get("login_result")
|
|
845
|
+
oauth_findings = intel.get("oauth_findings", [])
|
|
846
|
+
captcha_findings = intel.get("captcha_findings", [])
|
|
847
|
+
|
|
848
|
+
if login_result is not None:
|
|
849
|
+
ok = login_result.get("ok")
|
|
850
|
+
emit.section("LOGIN AUTOMATION", orbital=True)
|
|
851
|
+
if nc:
|
|
852
|
+
print(f" [{'OK' if ok else 'FAILED'}] " + (
|
|
853
|
+
f"Authenticated — cookies set: {', '.join(login_result.get('cookies_set', []))}"
|
|
854
|
+
if ok else f"Not authenticated — {login_result.get('reason','unknown reason')}"))
|
|
855
|
+
else:
|
|
856
|
+
if ok:
|
|
857
|
+
print(f" {C.G}●{C.RST} {C.G}Authenticated{C.RST} "
|
|
858
|
+
f"{C.W}cookies set: {', '.join(login_result.get('cookies_set', []))}{C.RST}")
|
|
859
|
+
else:
|
|
860
|
+
print(f" {C.R}●{C.RST} {C.R}Not authenticated{C.RST} "
|
|
861
|
+
f"{C.GR}{login_result.get('reason','unknown reason')}{C.RST}")
|
|
862
|
+
|
|
863
|
+
if oauth_findings:
|
|
864
|
+
emit.section(f"OAUTH / SSO DETECTION ({len(oauth_findings)} redirect(s) fingerprinted)", orbital=True)
|
|
865
|
+
if not nc:
|
|
866
|
+
emit.row("Note", "Detected during unauthenticated recon — fingerprinting only, "
|
|
867
|
+
"Spider never follows or automates the OAuth flow itself",
|
|
868
|
+
icon="○", label_colour=C.CY, value_colour=C.W)
|
|
869
|
+
for f in sorted(oauth_findings, key=lambda f: f.get("provider","")):
|
|
870
|
+
provider = f.get("provider","Unknown")
|
|
871
|
+
trig = f.get("trigger_url","")
|
|
872
|
+
authz = f.get("authorize_url","")
|
|
873
|
+
params = f.get("params", {})
|
|
874
|
+
gaps = []
|
|
875
|
+
if not f.get("has_pkce"): gaps.append("no PKCE")
|
|
876
|
+
if not f.get("has_state"): gaps.append("no state param")
|
|
877
|
+
gap_str = f" [{', '.join(gaps)}]" if gaps else ""
|
|
878
|
+
if nc:
|
|
879
|
+
print(f" ● {provider:<28} {trig} -> {authz}{gap_str}")
|
|
880
|
+
if params:
|
|
881
|
+
print(f" params: {params}")
|
|
882
|
+
else:
|
|
883
|
+
gap_col = C.R if gaps else C.G
|
|
884
|
+
print(f" {C.MG}●{C.RST} {C.W}{provider:<28}{C.RST} {C.CYD}{trig}{C.RST}")
|
|
885
|
+
print(f" {C.GR} └─ authorize:{C.RST} {C.Y}{authz}{C.RST}{f' {gap_col}{gap_str}{C.RST}' if gap_str else ''}")
|
|
886
|
+
if params:
|
|
887
|
+
pstr = ", ".join(f"{k}={v}" for k, v in params.items())
|
|
888
|
+
print(f" {C.GR} params: {pstr}{C.RST}")
|
|
889
|
+
|
|
890
|
+
if captcha_findings:
|
|
891
|
+
emit.section(f"CAPTCHA DETECTION ({len(captcha_findings)} found)", orbital=True)
|
|
892
|
+
if not nc:
|
|
893
|
+
emit.row("Note", "Detect-and-report only — Spider never attempts to solve a CAPTCHA",
|
|
894
|
+
icon="○", label_colour=C.CY, value_colour=C.W)
|
|
895
|
+
for f in sorted(captcha_findings, key=lambda f: f.get("url","")):
|
|
896
|
+
if nc:
|
|
897
|
+
print(f" ● {f.get('type',''):<28} {f.get('url','')}")
|
|
898
|
+
else:
|
|
899
|
+
print(f" {C.Y}●{C.RST} {C.W}{f.get('type',''):<28}{C.RST} {C.CYD}{f.get('url','')}{C.RST}")
|
|
900
|
+
|
|
901
|
+
|
|
841
902
|
crt_subs = intel.get("crt_subdomains", [])
|
|
842
903
|
if crt_subs:
|
|
843
904
|
emit.section(f"CRT.SH SUBDOMAINS ({len(crt_subs)} discovered)", orbital=True)
|
|
@@ -1699,6 +1760,13 @@ class Config:
|
|
|
1699
1760
|
# default; disable with --no-backtrack.
|
|
1700
1761
|
self.enable_backtrack = kw.get("enable_backtrack", True)
|
|
1701
1762
|
self.ctf_flag_templates = kw.get("ctf_flag_templates", [])
|
|
1763
|
+
# Classic HTML username/password form-login automation. Off unless
|
|
1764
|
+
# --login is given. OAuth/SSO redirect detection and CAPTCHA
|
|
1765
|
+
# detection run as part of it, and also run independently during
|
|
1766
|
+
# ordinary unauthenticated recon regardless of this flag.
|
|
1767
|
+
self.login_url = kw.get("login_url", None)
|
|
1768
|
+
self.login_user = kw.get("login_user", None)
|
|
1769
|
+
self.login_pass = kw.get("login_pass", None)
|
|
1702
1770
|
|
|
1703
1771
|
self.extra_scope: frozenset = frozenset(kw.get("extra_scope", []))
|
|
1704
1772
|
self.user_agent = kw.get(
|
|
@@ -1714,6 +1782,10 @@ class Config:
|
|
|
1714
1782
|
raise ValueError("concurrency must be 1–100")
|
|
1715
1783
|
if self.max_runtime is not None and self.max_runtime < 0:
|
|
1716
1784
|
raise ValueError("max_runtime must be >= 0 (0 disables the cap)")
|
|
1785
|
+
if self.login_url and not (self.login_user and self.login_pass):
|
|
1786
|
+
raise ValueError("--login requires both --login-user and --login-pass")
|
|
1787
|
+
if (self.login_user or self.login_pass) and not self.login_url:
|
|
1788
|
+
raise ValueError("--login-user/--login-pass require --login <url>")
|
|
1717
1789
|
|
|
1718
1790
|
|
|
1719
1791
|
|
|
@@ -1903,9 +1975,120 @@ async def fetch_with_redirect(session, method, url, rl, max_retries=3, base_dela
|
|
|
1903
1975
|
await asyncio.sleep(base_delay * (2**attempt))
|
|
1904
1976
|
return None, None, None, url
|
|
1905
1977
|
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1978
|
+
|
|
1979
|
+
async def fetch_no_redirect(session, method, url, rl, max_retries=2, base_delay=0.5, **kw):
|
|
1980
|
+
"""Like fetch(), but never follows redirects — returns the raw Location
|
|
1981
|
+
header too. Used for OAuth/SSO redirect fingerprinting, where the whole
|
|
1982
|
+
point is to see where the target *would* have sent the browser, not to
|
|
1983
|
+
follow it there."""
|
|
1984
|
+
domain = urlparse(url).netloc
|
|
1985
|
+
await rl.wait(domain)
|
|
1986
|
+
for attempt in range(max_retries + 1):
|
|
1987
|
+
try:
|
|
1988
|
+
async with session.request(method, url, ssl=False,
|
|
1989
|
+
allow_redirects=False, **kw) as resp:
|
|
1990
|
+
if resp.status == 429:
|
|
1991
|
+
rl.backoff(domain)
|
|
1992
|
+
await asyncio.sleep(float(resp.headers.get("Retry-After", base_delay * (2**attempt))))
|
|
1993
|
+
continue
|
|
1994
|
+
body = ""
|
|
1995
|
+
try:
|
|
1996
|
+
body = await resp.text(errors="replace")
|
|
1997
|
+
except Exception:
|
|
1998
|
+
body = ""
|
|
1999
|
+
rl.recover(domain)
|
|
2000
|
+
return resp.status, dict(resp.headers), body, resp.headers.get("Location", "")
|
|
2001
|
+
except Exception:
|
|
2002
|
+
if attempt < max_retries:
|
|
2003
|
+
await asyncio.sleep(base_delay * (2**attempt))
|
|
2004
|
+
return None, None, "", ""
|
|
2005
|
+
|
|
2006
|
+
|
|
2007
|
+
# ── CAPTCHA detection (detect-and-report only — Spider never attempts to
|
|
2008
|
+
# solve or bypass a CAPTCHA) ─────────────────────────────────────────────
|
|
2009
|
+
_CAPTCHA_SIGNATURES = (
|
|
2010
|
+
(re.compile(r'g-recaptcha|recaptcha/api\.js|grecaptcha\.render|www\.google\.com/recaptcha', re.I), "reCAPTCHA (Google)"),
|
|
2011
|
+
(re.compile(r'h-captcha|hcaptcha\.com/1/api\.js', re.I), "hCaptcha"),
|
|
2012
|
+
(re.compile(r'cf-turnstile|challenges\.cloudflare\.com/turnstile', re.I), "Cloudflare Turnstile"),
|
|
2013
|
+
(re.compile(r'arkoselabs|funcaptcha|fc-token', re.I), "Arkose Labs / FunCaptcha"),
|
|
2014
|
+
(re.compile(r'captcha', re.I), "Generic/Unidentified CAPTCHA"),
|
|
2015
|
+
)
|
|
2016
|
+
|
|
2017
|
+
def detect_captcha(body: Optional[str]) -> Optional[str]:
|
|
2018
|
+
"""Return a human-readable CAPTCHA provider name if the response body
|
|
2019
|
+
carries a recognizable CAPTCHA widget marker, else None. Detection only."""
|
|
2020
|
+
if not body:
|
|
2021
|
+
return None
|
|
2022
|
+
for pattern, name in _CAPTCHA_SIGNATURES:
|
|
2023
|
+
if pattern.search(body):
|
|
2024
|
+
return name
|
|
2025
|
+
return None
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
# ── OAuth / SSO redirect fingerprinting (detect-and-report only — Spider
|
|
2029
|
+
# never attempts to automate or bypass the actual OAuth/OIDC flow) ───────
|
|
2030
|
+
_OAUTH_LOGIN_TRIGGER_PATHS = (
|
|
2031
|
+
"/login", "/signin", "/sign-in", "/log-in", "/account/login",
|
|
2032
|
+
"/user/login", "/auth", "/auth/login", "/authorize", "/authenticate",
|
|
2033
|
+
"/sso", "/sso/login", "/oauth", "/oauth2", "/connect/authorize",
|
|
2034
|
+
"/dashboard", "/account", "/admin",
|
|
2035
|
+
)
|
|
2036
|
+
|
|
2037
|
+
_OAUTH_PROVIDER_SIGS = (
|
|
2038
|
+
(re.compile(r'\.okta\.com|\.oktapreview\.com|\.okta-emea\.com', re.I), "Okta"),
|
|
2039
|
+
(re.compile(r'login\.microsoftonline\.com|\.b2clogin\.com|login\.windows\.net|login\.live\.com', re.I), "Microsoft Entra ID / Azure AD"),
|
|
2040
|
+
(re.compile(r'accounts\.google\.com', re.I), "Google"),
|
|
2041
|
+
(re.compile(r'\.auth0\.com', re.I), "Auth0"),
|
|
2042
|
+
(re.compile(r'\.onelogin\.com', re.I), "OneLogin"),
|
|
2043
|
+
(re.compile(r'\.pingone\.com|\.pingidentity\.com', re.I), "Ping Identity"),
|
|
2044
|
+
(re.compile(r'cognito-idp\.[\w-]+\.amazonaws\.com|\.auth\.[\w-]+\.amazoncognito\.com', re.I), "AWS Cognito"),
|
|
2045
|
+
(re.compile(r'/realms/[^/]+/protocol/openid-connect', re.I), "Keycloak"),
|
|
2046
|
+
(re.compile(r'github\.com/login/oauth', re.I), "GitHub OAuth"),
|
|
2047
|
+
(re.compile(r'(?:www\.)?facebook\.com/.*dialog/oauth', re.I), "Facebook OAuth"),
|
|
2048
|
+
(re.compile(r'appleid\.apple\.com', re.I), "Sign in with Apple"),
|
|
2049
|
+
(re.compile(r'\.frontegg\.com', re.I), "Frontegg"),
|
|
2050
|
+
)
|
|
2051
|
+
|
|
2052
|
+
# Shape of an OAuth2/OIDC authorization request — a query string carrying
|
|
2053
|
+
# client_id/response_type, or a path that looks like an /authorize endpoint.
|
|
2054
|
+
_OAUTH_QS_SHAPE_RE = re.compile(r'[?&](client_id|response_type)=', re.I)
|
|
2055
|
+
_OAUTH_PATH_SHAPE_RE = re.compile(r'/(?:oauth2?/)?(?:authorize|auth)(?:[/?]|$)|/connect/authorize|/protocol/openid-connect/auth', re.I)
|
|
2056
|
+
|
|
2057
|
+
_OAUTH_PARAM_KEYS = (
|
|
2058
|
+
"client_id", "redirect_uri", "response_type", "scope", "state",
|
|
2059
|
+
"code_challenge", "code_challenge_method", "nonce", "response_mode",
|
|
2060
|
+
)
|
|
2061
|
+
|
|
2062
|
+
def classify_oauth_redirect(location: str) -> Optional[dict]:
|
|
2063
|
+
"""Given an absolute URL a target redirected to, decide whether it looks
|
|
2064
|
+
like a third-party OAuth2/OIDC authorization request and, if so, which
|
|
2065
|
+
provider it is. Returns None for anything that doesn't look OAuth-shaped
|
|
2066
|
+
(e.g. a plain same-site redirect to another page). This is pure
|
|
2067
|
+
fingerprinting — it never follows the URL or attempts the flow."""
|
|
2068
|
+
if not location:
|
|
2069
|
+
return None
|
|
2070
|
+
looks_oauth = bool(_OAUTH_QS_SHAPE_RE.search(location)) or bool(_OAUTH_PATH_SHAPE_RE.search(location))
|
|
2071
|
+
if not looks_oauth:
|
|
2072
|
+
return None
|
|
2073
|
+
provider = "Generic OAuth2/OIDC"
|
|
2074
|
+
for pattern, name in _OAUTH_PROVIDER_SIGS:
|
|
2075
|
+
if pattern.search(location):
|
|
2076
|
+
provider = name
|
|
2077
|
+
break
|
|
2078
|
+
parsed = urlparse(location)
|
|
2079
|
+
qs = parse_qs(parsed.query)
|
|
2080
|
+
params = {k: qs[k][0] for k in _OAUTH_PARAM_KEYS if k in qs}
|
|
2081
|
+
return {
|
|
2082
|
+
"provider": provider,
|
|
2083
|
+
"authorize_url": location,
|
|
2084
|
+
"params": params,
|
|
2085
|
+
"has_pkce": "code_challenge" in params,
|
|
2086
|
+
"has_state": "state" in params,
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
|
|
2090
|
+
|
|
2091
|
+
|
|
1909
2092
|
|
|
1910
2093
|
_ID_RE = re.compile(
|
|
1911
2094
|
r'^(?:\d{1,20}'
|
|
@@ -2165,6 +2348,15 @@ class Store:
|
|
|
2165
2348
|
self.cloud_probes: List[dict] = []
|
|
2166
2349
|
self.page_bg_requests: Dict[str, List[dict]] = defaultdict(list)
|
|
2167
2350
|
self._bg_req_seen: Set[tuple] = set()
|
|
2351
|
+
# OAuth/SSO redirect fingerprints found during unauthenticated recon,
|
|
2352
|
+
# CAPTCHA widget markers found on any probed page, and the outcome
|
|
2353
|
+
# (if any) of --login form-login automation. All detect/report —
|
|
2354
|
+
# nothing here is ever acted on automatically.
|
|
2355
|
+
self.oauth_findings: List[dict] = []
|
|
2356
|
+
self._oauth_seen: Set[str] = set()
|
|
2357
|
+
self.captcha_findings: List[dict] = []
|
|
2358
|
+
self._captcha_seen: Set[str] = set()
|
|
2359
|
+
self.login_result: Optional[dict] = None
|
|
2168
2360
|
|
|
2169
2361
|
def record_bg_request(self, page_url: str, bg_url: str, method: str = "GET", source: str = "DeepSPA"):
|
|
2170
2362
|
if not page_url or not bg_url:
|
|
@@ -2853,6 +3045,9 @@ class Store:
|
|
|
2853
3045
|
"js_libs": self.js_libs,
|
|
2854
3046
|
"cloud_probes": self.cloud_probes,
|
|
2855
3047
|
"spa_background_requests": dict(self.page_bg_requests),
|
|
3048
|
+
"oauth_findings": self.oauth_findings,
|
|
3049
|
+
"captcha_findings": self.captcha_findings,
|
|
3050
|
+
"login_result": self.login_result,
|
|
2856
3051
|
|
|
2857
3052
|
|
|
2858
3053
|
|
|
@@ -8572,6 +8767,211 @@ class Spider:
|
|
|
8572
8767
|
auth_required=True)
|
|
8573
8768
|
self.emit.always_success(f"[OIDC] {key}: {ep_url}")
|
|
8574
8769
|
|
|
8770
|
+
async def _record_oauth_finding(self, trigger_url: str, info: dict):
|
|
8771
|
+
if info["authorize_url"] in self.store._oauth_seen:
|
|
8772
|
+
return
|
|
8773
|
+
self.store.oauth_findings.append({"trigger_url": trigger_url, **info})
|
|
8774
|
+
self.store._oauth_seen.add(info["authorize_url"])
|
|
8775
|
+
self.emit.crawl_feed("OAuth/SSO", "GET", Emit.trunc_url(info["authorize_url"]))
|
|
8776
|
+
self.emit.always_success(f"[OAuth] {info['provider']} detected ← {Emit.trunc_url(trigger_url, 50)}")
|
|
8777
|
+
gaps = []
|
|
8778
|
+
if not info["has_pkce"]:
|
|
8779
|
+
gaps.append("no PKCE (code_challenge)")
|
|
8780
|
+
if not info["has_state"]:
|
|
8781
|
+
gaps.append("no state param (possible CSRF on the OAuth callback)")
|
|
8782
|
+
if gaps:
|
|
8783
|
+
self.emit.always_info(f"[OAuth] {info['provider']} authorize request missing: {', '.join(gaps)}")
|
|
8784
|
+
|
|
8785
|
+
def _record_captcha_finding(self, url: str, kind: str):
|
|
8786
|
+
if url in self.store._captcha_seen:
|
|
8787
|
+
return
|
|
8788
|
+
self.store._captcha_seen.add(url)
|
|
8789
|
+
self.store.captcha_findings.append({"url": url, "type": kind})
|
|
8790
|
+
self.emit.always_info(f"[CAPTCHA] {kind} detected at {Emit.trunc_url(url, 55)}")
|
|
8791
|
+
|
|
8792
|
+
async def _detect_oauth_sso(self, session):
|
|
8793
|
+
"""Unauthenticated-recon OAuth/SSO fingerprinting. Probes a short list
|
|
8794
|
+
of common login-trigger paths with redirects disabled so the raw
|
|
8795
|
+
Location header (if any) can be inspected, classifies anything
|
|
8796
|
+
OAuth/OIDC-authorize-shaped, and identifies the provider by hostname/
|
|
8797
|
+
path pattern. Also checks whatever body comes back for CAPTCHA
|
|
8798
|
+
markers. Detection and fingerprinting only — Spider never attempts
|
|
8799
|
+
to follow or automate the actual OAuth flow, and never attempts to
|
|
8800
|
+
solve a CAPTCHA. Runs during every unauthenticated recon pass,
|
|
8801
|
+
independent of --login."""
|
|
8802
|
+
for path in _OAUTH_LOGIN_TRIGGER_PATHS:
|
|
8803
|
+
url = urljoin(self.target, path)
|
|
8804
|
+
try:
|
|
8805
|
+
status, headers, body, loc = await fetch_no_redirect(session, "GET", url, self.rl)
|
|
8806
|
+
except Exception:
|
|
8807
|
+
continue
|
|
8808
|
+
if status is None:
|
|
8809
|
+
continue
|
|
8810
|
+
|
|
8811
|
+
if loc:
|
|
8812
|
+
abs_loc = urljoin(url, loc)
|
|
8813
|
+
info = classify_oauth_redirect(abs_loc)
|
|
8814
|
+
if info:
|
|
8815
|
+
await self._record_oauth_finding(url, info)
|
|
8816
|
+
|
|
8817
|
+
cap = detect_captcha(body)
|
|
8818
|
+
if cap:
|
|
8819
|
+
self._record_captcha_finding(url, cap)
|
|
8820
|
+
|
|
8821
|
+
async def _attempt_login(self, session):
|
|
8822
|
+
"""Classic HTML username/password form-login automation, gated
|
|
8823
|
+
behind --login/--login-user/--login-pass. Parses the login form,
|
|
8824
|
+
pre-checks the page for an OAuth/SSO redirect or a CAPTCHA and
|
|
8825
|
+
aborts (reporting, not attempting) if either is found — those are
|
|
8826
|
+
both explicitly out of scope for automation — then submits the
|
|
8827
|
+
form and merges any resulting session cookies into the active
|
|
8828
|
+
aiohttp session so the rest of the scan runs authenticated."""
|
|
8829
|
+
login_url = self.cfg.login_url
|
|
8830
|
+
self.emit.animator.update(0, "Recon Login")
|
|
8831
|
+
self.emit.always_info(f"[Login] Attempting form login at {login_url}")
|
|
8832
|
+
|
|
8833
|
+
status, headers, body, loc = await fetch_no_redirect(session, "GET", login_url, self.rl)
|
|
8834
|
+
if status is None:
|
|
8835
|
+
self.store.login_result = {"ok": False, "reason": "login page unreachable"}
|
|
8836
|
+
self.emit.warn(f"[Login] {login_url} unreachable — skipping login automation")
|
|
8837
|
+
return
|
|
8838
|
+
|
|
8839
|
+
# Target already redirected before we even saw a form — almost
|
|
8840
|
+
# certainly SSO, not a page we can submit credentials to.
|
|
8841
|
+
if loc:
|
|
8842
|
+
abs_loc = urljoin(login_url, loc)
|
|
8843
|
+
info = classify_oauth_redirect(abs_loc)
|
|
8844
|
+
if info:
|
|
8845
|
+
await self._record_oauth_finding(login_url, info)
|
|
8846
|
+
self.store.login_result = {
|
|
8847
|
+
"ok": False,
|
|
8848
|
+
"reason": f"login URL redirects to {info['provider']} (OAuth/SSO) — "
|
|
8849
|
+
f"automated form submission isn't applicable here",
|
|
8850
|
+
}
|
|
8851
|
+
self.emit.warn(f"[Login] Aborted — {login_url} redirects to an OAuth/SSO "
|
|
8852
|
+
f"provider ({info['provider']}). Use --cookie with a "
|
|
8853
|
+
f"manually-obtained session instead.")
|
|
8854
|
+
return
|
|
8855
|
+
# Non-OAuth redirect (e.g. http->https, or a login page that
|
|
8856
|
+
# lives at a different path) — follow it once to find the form.
|
|
8857
|
+
status, headers, body, _ = await fetch(session, "GET", abs_loc, self.rl)
|
|
8858
|
+
login_url = abs_loc
|
|
8859
|
+
if status is None:
|
|
8860
|
+
self.store.login_result = {"ok": False, "reason": "login page unreachable after redirect"}
|
|
8861
|
+
return
|
|
8862
|
+
|
|
8863
|
+
cap = detect_captcha(body)
|
|
8864
|
+
if cap:
|
|
8865
|
+
self._record_captcha_finding(login_url, cap)
|
|
8866
|
+
self.store.login_result = {"ok": False, "reason": f"{cap} present on login page"}
|
|
8867
|
+
self.emit.warn(f"[Login] Aborted — {cap} detected on {login_url}. "
|
|
8868
|
+
f"Spider detects and reports CAPTCHAs; it never attempts to solve them.")
|
|
8869
|
+
return
|
|
8870
|
+
|
|
8871
|
+
soup = BeautifulSoup(body or "", "lxml")
|
|
8872
|
+
login_form = None
|
|
8873
|
+
for form in soup.find_all("form"):
|
|
8874
|
+
if form.find("input", attrs={"type": re.compile("^password$", re.I)}):
|
|
8875
|
+
login_form = form
|
|
8876
|
+
break
|
|
8877
|
+
if login_form is None:
|
|
8878
|
+
self.store.login_result = {"ok": False, "reason": "no username/password form found on the page"}
|
|
8879
|
+
self.emit.warn(f"[Login] Aborted — no login form (password field) found at {login_url}")
|
|
8880
|
+
return
|
|
8881
|
+
|
|
8882
|
+
action = login_form.get("action") or login_url
|
|
8883
|
+
post_url = urljoin(login_url, action)
|
|
8884
|
+
method = (login_form.get("method") or "POST").upper()
|
|
8885
|
+
|
|
8886
|
+
payload = {}
|
|
8887
|
+
user_field = pass_field = None
|
|
8888
|
+
for el in login_form.find_all(["input", "select", "textarea"]):
|
|
8889
|
+
nm = (el.get("name") or "").strip()
|
|
8890
|
+
if not nm:
|
|
8891
|
+
continue
|
|
8892
|
+
el_type = (el.get("type") or "text").lower()
|
|
8893
|
+
if el_type == "password":
|
|
8894
|
+
pass_field = nm
|
|
8895
|
+
continue
|
|
8896
|
+
if el_type in ("submit", "button", "reset", "image", "file"):
|
|
8897
|
+
continue
|
|
8898
|
+
if el_type == "hidden":
|
|
8899
|
+
payload[nm] = el.get("value", "")
|
|
8900
|
+
continue
|
|
8901
|
+
# Likely the username/email field — first non-hidden text-ish
|
|
8902
|
+
# input that isn't the password field itself.
|
|
8903
|
+
name_id_blob = f"{nm} {el.get('id','')} {el.get('placeholder','')}".lower()
|
|
8904
|
+
if user_field is None and (
|
|
8905
|
+
el_type in ("text", "email") or
|
|
8906
|
+
re.search(r'user|email|login|account', name_id_blob)
|
|
8907
|
+
):
|
|
8908
|
+
user_field = nm
|
|
8909
|
+
else:
|
|
8910
|
+
payload.setdefault(nm, el.get("value", ""))
|
|
8911
|
+
|
|
8912
|
+
if not pass_field:
|
|
8913
|
+
self.store.login_result = {"ok": False, "reason": "form has no password field"}
|
|
8914
|
+
self.emit.warn(f"[Login] Aborted — couldn't identify a password field in the form at {login_url}")
|
|
8915
|
+
return
|
|
8916
|
+
if not user_field:
|
|
8917
|
+
# Fall back to the first non-password, non-hidden input.
|
|
8918
|
+
for el in login_form.find_all(["input"]):
|
|
8919
|
+
nm = (el.get("name") or "").strip()
|
|
8920
|
+
el_type = (el.get("type") or "text").lower()
|
|
8921
|
+
if nm and nm != pass_field and el_type not in ("hidden", "submit", "button", "reset", "image", "file"):
|
|
8922
|
+
user_field = nm
|
|
8923
|
+
break
|
|
8924
|
+
if not user_field:
|
|
8925
|
+
self.store.login_result = {"ok": False, "reason": "couldn't identify a username/email field"}
|
|
8926
|
+
self.emit.warn(f"[Login] Aborted — couldn't identify a username field in the form at {login_url}")
|
|
8927
|
+
return
|
|
8928
|
+
|
|
8929
|
+
payload[user_field] = self.cfg.login_user
|
|
8930
|
+
payload[pass_field] = self.cfg.login_pass
|
|
8931
|
+
|
|
8932
|
+
pre_login_cookies = {c.key: c.value for c in session.cookie_jar}
|
|
8933
|
+
try:
|
|
8934
|
+
async with session.request(method, post_url, data=payload, ssl=False,
|
|
8935
|
+
allow_redirects=True,
|
|
8936
|
+
timeout=aiohttp.ClientTimeout(total=self.cfg.timeout)) as resp:
|
|
8937
|
+
resp_status = resp.status
|
|
8938
|
+
resp_text = await resp.text(errors="replace")
|
|
8939
|
+
final_url = str(resp.url)
|
|
8940
|
+
except Exception as e:
|
|
8941
|
+
self.store.login_result = {"ok": False, "reason": f"submission failed: {e}"}
|
|
8942
|
+
self.emit.warn(f"[Login] Submission to {post_url} failed: {e}")
|
|
8943
|
+
return
|
|
8944
|
+
|
|
8945
|
+
post_login_cookies = {c.key: c.value for c in session.cookie_jar}
|
|
8946
|
+
new_cookies = {k: v for k, v in post_login_cookies.items() if pre_login_cookies.get(k) != v}
|
|
8947
|
+
|
|
8948
|
+
# Heuristic success check: got new cookies beyond what we started
|
|
8949
|
+
# with, and nothing on the response page still smells like a fresh
|
|
8950
|
+
# login form (password field) or an explicit error marker.
|
|
8951
|
+
still_has_password_field = bool(re.search(
|
|
8952
|
+
r'type=["\']?password', resp_text or "", re.I))
|
|
8953
|
+
looks_like_error = bool(re.search(
|
|
8954
|
+
r'invalid (?:username|password|login|credentials)|login failed|incorrect password',
|
|
8955
|
+
resp_text or "", re.I))
|
|
8956
|
+
success = bool(new_cookies) and not still_has_password_field and not looks_like_error
|
|
8957
|
+
|
|
8958
|
+
if success:
|
|
8959
|
+
self.cookies.update(new_cookies)
|
|
8960
|
+
self.store.login_result = {
|
|
8961
|
+
"ok": True, "post_url": post_url, "final_url": final_url,
|
|
8962
|
+
"cookies_set": sorted(new_cookies.keys()),
|
|
8963
|
+
}
|
|
8964
|
+
self.emit.always_success(
|
|
8965
|
+
f"[Login] Success — {len(new_cookies)} session cookie(s) set, "
|
|
8966
|
+
f"continuing the scan authenticated")
|
|
8967
|
+
else:
|
|
8968
|
+
reason = "server returned a login error" if looks_like_error else \
|
|
8969
|
+
"no new session cookie after submission" if not new_cookies else \
|
|
8970
|
+
"response still looks like the login page"
|
|
8971
|
+
self.store.login_result = {"ok": False, "reason": reason, "status": resp_status, "final_url": final_url}
|
|
8972
|
+
self.emit.warn(f"[Login] Submission to {post_url} didn't look successful ({reason}) — "
|
|
8973
|
+
f"continuing the scan unauthenticated")
|
|
8974
|
+
|
|
8575
8975
|
_AUTOINDEX_SIGS = re.compile(
|
|
8576
8976
|
r'index of /|<title>\s*index of|parent directory</a>|\[to parent directory\]|'
|
|
8577
8977
|
r'directory listing for',
|
|
@@ -8735,8 +9135,14 @@ class Spider:
|
|
|
8735
9135
|
req_headers.update(self.extra_headers)
|
|
8736
9136
|
connector = aiohttp.TCPConnector(limit=self.cfg.concurrency, ttl_dns_cache=300, ssl=False)
|
|
8737
9137
|
timeout = aiohttp.ClientTimeout(total=self.cfg.timeout)
|
|
9138
|
+
# unsafe=True: aiohttp's default cookie jar silently refuses to store
|
|
9139
|
+
# cookies for plain IP-address hosts, which breaks session-cookie
|
|
9140
|
+
# handling (including --login) against bare-IP and internal targets —
|
|
9141
|
+
# a routine case in pentesting.
|
|
9142
|
+
cookie_jar = aiohttp.CookieJar(unsafe=True)
|
|
8738
9143
|
async with aiohttp.ClientSession(headers=req_headers, cookies=self.cookies,
|
|
8739
|
-
timeout=timeout, connector=connector
|
|
9144
|
+
timeout=timeout, connector=connector,
|
|
9145
|
+
cookie_jar=cookie_jar) as session:
|
|
8740
9146
|
try:
|
|
8741
9147
|
|
|
8742
9148
|
_t_phase_start = time.time()
|
|
@@ -8773,6 +9179,27 @@ class Spider:
|
|
|
8773
9179
|
|
|
8774
9180
|
self.emit.animator.start_anim("Recon Probing Base")
|
|
8775
9181
|
|
|
9182
|
+
# ── Auth surface: optional form-login automation, then
|
|
9183
|
+
# OAuth/SSO redirect fingerprinting + CAPTCHA detection ──
|
|
9184
|
+
# Login runs first (when requested) so its cookies are
|
|
9185
|
+
# merged into the session before anything else. The OAuth/
|
|
9186
|
+
# SSO probe runs unconditionally afterward — on a target
|
|
9187
|
+
# that redirects to third-party SSO, the unauthenticated
|
|
9188
|
+
# case is exactly when these login-trigger paths bounce
|
|
9189
|
+
# somewhere interesting; a successfully authenticated scan
|
|
9190
|
+
# simply won't see those redirects, so this is safe either way.
|
|
9191
|
+
if self.cfg.login_url:
|
|
9192
|
+
try:
|
|
9193
|
+
await self._attempt_login(session)
|
|
9194
|
+
except Exception as e:
|
|
9195
|
+
self.store.login_result = {"ok": False, "reason": f"unexpected error: {e}"}
|
|
9196
|
+
self.emit.warn(f"[Login] Unexpected error during login automation: {e}")
|
|
9197
|
+
self.emit.animator.update(0, "Recon Auth / SSO Surface")
|
|
9198
|
+
try:
|
|
9199
|
+
await self._detect_oauth_sso(session)
|
|
9200
|
+
except Exception as e:
|
|
9201
|
+
self.emit.warn(f"[OAuth] Detection pass failed: {e}")
|
|
9202
|
+
|
|
8776
9203
|
# ── Fire WhatWeb + TLS concurrently ───────────────────────────
|
|
8777
9204
|
async def _tls_and_headers():
|
|
8778
9205
|
if not getattr(self.cfg, 'no_tls', False):
|
|
@@ -9363,8 +9790,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
9363
9790
|
auth = p.add_argument_group(f"{C.CY}Authentication{C.RST}")
|
|
9364
9791
|
auth.add_argument("--cookie", "-C", type=str, default=None, metavar="COOKIE",
|
|
9365
9792
|
help='Cookie string (e.g., "session=42"), dict, or path to a cookie file. '
|
|
9366
|
-
'
|
|
9367
|
-
'
|
|
9793
|
+
'For an OAuth/SSO-gated target, or any login flow --login can\'t handle, '
|
|
9794
|
+
'log in manually via a browser and supply the resulting session cookie here instead.')
|
|
9368
9795
|
auth.add_argument("--auth", "-a", type=str, default=None, metavar="HEADER",
|
|
9369
9796
|
help='Authorization header e.g. "Bearer eyJ..."')
|
|
9370
9797
|
auth.add_argument("--basic-auth", "-u", type=str, default=None, metavar="USER:PASS",
|
|
@@ -9374,6 +9801,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
9374
9801
|
help='Custom header, repeatable, formatted as "Name: Value" e.g. -X "X-Bug-Bounty: Bugcrowd-yourhandle" '
|
|
9375
9802
|
'-X "X-Research-Purpose: authorized-pentest" '
|
|
9376
9803
|
'(use for program-required tester-identification headers)')
|
|
9804
|
+
auth.add_argument("--login", "-l", type=str, default=None, metavar="URL",
|
|
9805
|
+
help='URL of a classic HTML username/password login form. Spider parses the '
|
|
9806
|
+
'form, submits -n/-z (--login-user/--login-pass), and carries the resulting '
|
|
9807
|
+
'session cookies through the rest of the scan. Aborts with a clear report '
|
|
9808
|
+
'(no submission attempted) if the page redirects to a third-party OAuth/SSO '
|
|
9809
|
+
'provider or serves a CAPTCHA — neither is something Spider will try to bypass; '
|
|
9810
|
+
'use --cookie instead for those.')
|
|
9811
|
+
auth.add_argument("--login-user", "-n", type=str, default=None, metavar="USERNAME",
|
|
9812
|
+
help='Username/email to submit with -l/--login')
|
|
9813
|
+
auth.add_argument("--login-pass", "-z", type=str, default=None, metavar="PASSWORD",
|
|
9814
|
+
help='Password to submit with -l/--login')
|
|
9377
9815
|
|
|
9378
9816
|
out = p.add_argument_group(f"{C.CY}Output{C.RST}")
|
|
9379
9817
|
out.add_argument("--out", "-o", type=str, default=None, metavar="FILE",
|
|
@@ -9683,6 +10121,9 @@ def main():
|
|
|
9683
10121
|
if args.ctf_flag else [],
|
|
9684
10122
|
extra_scope = [h.strip() for h in args.scope.split(",") if h.strip()]
|
|
9685
10123
|
if args.scope else [],
|
|
10124
|
+
login_url = args.login,
|
|
10125
|
+
login_user = args.login_user,
|
|
10126
|
+
login_pass = args.login_pass,
|
|
9686
10127
|
)
|
|
9687
10128
|
|
|
9688
10129
|
print()
|
|
File without changes
|
|
File without changes
|
{hellhound_spider-14.2 → hellhound_spider-14.3}/hellhound_spider.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|