xssmap 2.9.0__tar.gz → 2.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xssmap-2.9.0 → xssmap-2.9.1}/CHANGELOG.md +28 -4
- {xssmap-2.9.0/xssmap.egg-info → xssmap-2.9.1}/PKG-INFO +24 -4
- {xssmap-2.9.0 → xssmap-2.9.1}/README.md +23 -3
- xssmap-2.9.1/tests/test_speed.py +114 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/__init__.py +1 -1
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/cli.py +16 -3
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/crawler.py +6 -2
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/scanner.py +58 -11
- {xssmap-2.9.0 → xssmap-2.9.1/xssmap.egg-info}/PKG-INFO +24 -4
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/SOURCES.txt +1 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/LICENSE +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/MANIFEST.in +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/pyproject.toml +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/setup.cfg +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_bench.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_brute.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_budget.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_charset.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_crawlperf.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_csp.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_domscan.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_domstate.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_efficiency.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_exploit.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_fuzz.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_honesty.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_inline.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_jslib.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_leads.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_mxss.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_ops.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_params.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_recon_report.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_session.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_subdomains.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_tech.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_units.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_waf.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/__main__.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/browser.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/brute.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/callback.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/charset.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/contexts.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/csp.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/domscan.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/domstate.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/efficiency.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/exploit.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/filters.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/fuzz.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/htmlparse.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/jslib.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/mxss.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/net.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/params.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/payloads.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/py.typed +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/report.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/repro.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/sarif.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/session.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/state.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/stored.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/subdomains.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/tech.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/verify.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/waf.py +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/dependency_links.txt +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/entry_points.txt +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/requires.txt +0 -0
- {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/top_level.txt +0 -0
|
@@ -8,6 +8,29 @@ Releases before 2.8.0 predate this file; their history is in the git log.
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [2.9.1] - 2026-11-13
|
|
12
|
+
|
|
13
|
+
Speed release. Detection behaviour is unchanged.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Discovery has its own worker count. `--crawl-threads` defaults to 16 (or
|
|
18
|
+
`--threads` when that is larger) instead of running the crawl at the payload
|
|
19
|
+
width of 5; `--threads` itself now defaults to 10. On a 4,096-page lab target
|
|
20
|
+
with 50 ms of latency, the same 8,252 requests finished in 85 s before and
|
|
21
|
+
38 s after.
|
|
22
|
+
- The baseline response is fetched once per URL rather than once per parameter,
|
|
23
|
+
and concurrent workers share the in-flight fetch instead of racing on it.
|
|
24
|
+
Only the status is cached, never a body.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- Point order is deterministic. `param_urls` ties were broken by crawl
|
|
29
|
+
completion order, so `--max-points` could cut at a different place on every
|
|
30
|
+
run of the same target.
|
|
31
|
+
- A parameter that answers 500 when probed no longer re-sends the identical
|
|
32
|
+
probe; the error body is still analysed for a reflection.
|
|
33
|
+
|
|
11
34
|
## [2.9.0] - 2026-11-13
|
|
12
35
|
|
|
13
36
|
### Added
|
|
@@ -23,7 +46,7 @@ Releases before 2.8.0 predate this file; their history is in the git log.
|
|
|
23
46
|
8.2 (out of security support) and IIS 6.0-8.5 (shipped only with an
|
|
24
47
|
end-of-life Windows Server). A flagged version is a lead for the operator,
|
|
25
48
|
never a finding on its own.
|
|
26
|
-
-
|
|
49
|
+
- 36 tests pinning the new crawl scheduling, the transport cap, the throttle
|
|
27
50
|
rules and the stack policy table.
|
|
28
51
|
|
|
29
52
|
### Changed
|
|
@@ -43,9 +66,10 @@ Releases before 2.8.0 predate this file; their history is in the git log.
|
|
|
43
66
|
|
|
44
67
|
### Fixed
|
|
45
68
|
|
|
46
|
-
- Crawling a target that 500s part of its URL space is ~
|
|
47
|
-
page count:
|
|
48
|
-
(identical page sets,
|
|
69
|
+
- Crawling a target that 500s part of its URL space is ~20x faster at the same
|
|
70
|
+
page count: about 1 s instead of 20 s for 68 pages, measured by running the
|
|
71
|
+
2.8.1 tree and this one against the same local server (identical page sets,
|
|
72
|
+
50 ms of fixed latency, a deterministic 15% of URLs answering 500).
|
|
49
73
|
|
|
50
74
|
## [2.8.1] - 2026-11-13
|
|
51
75
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xssmap
|
|
3
|
-
Version: 2.9.
|
|
3
|
+
Version: 2.9.1
|
|
4
4
|
Summary: Context-aware XSS scanner: crawler, browser verification, mXSS, charset and stored-XSS stages
|
|
5
5
|
Author: Ian Mattas
|
|
6
6
|
License-Expression: MIT
|
|
@@ -511,6 +511,23 @@ request. Three things make the result usable rather than noisy:
|
|
|
511
511
|
A flagged version is a lead: it says the build has a published problem, not that this
|
|
512
512
|
host is exploitable through it. The finding standard for XSS is unchanged.
|
|
513
513
|
|
|
514
|
+
### Crawl and scan speed
|
|
515
|
+
|
|
516
|
+
Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
|
|
517
|
+
cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
|
|
518
|
+
pages with 50 ms of latency per response, the same code, same page set and the same
|
|
519
|
+
8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
|
|
520
|
+
the current ones** (16 discovery workers, 10 payload workers). At a real WAN
|
|
521
|
+
latency of 200-300 ms the difference is the same multiple, in minutes.
|
|
522
|
+
|
|
523
|
+
Discovery therefore has its own worker count. It only fetches pages the site
|
|
524
|
+
links to itself, and the shared throttle still backs off on 429/503, so it does
|
|
525
|
+
not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
|
|
526
|
+
`--threads` remains the knob for the scan phase. Two scan-side savings are
|
|
527
|
+
always on: the baseline response is fetched once per URL instead of once per
|
|
528
|
+
parameter, and the point order is deterministic, so `--max-points` cuts the same
|
|
529
|
+
places on every run.
|
|
530
|
+
|
|
514
531
|
### Browser cost
|
|
515
532
|
|
|
516
533
|
`--browser` reuses one browser and one page per run. The report's `browser_verification`
|
|
@@ -571,7 +588,8 @@ per check, so the price of execution proof is visible rather than implied.
|
|
|
571
588
|
| `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
|
|
572
589
|
| `--headless new\|old` | headless mode passed to the browser |
|
|
573
590
|
| `--screenshots` | capture a PNG for verified findings |
|
|
574
|
-
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
|
|
591
|
+
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
|
|
592
|
+
| `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
|
|
575
593
|
| `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
|
|
576
594
|
| `--max-points N` | stop after N injection points |
|
|
577
595
|
| `--state FILE` | resume file: skip points already completed, carry findings over |
|
|
@@ -767,7 +785,7 @@ response shape shows up as a miss on the other.
|
|
|
767
785
|
## Tests and CI
|
|
768
786
|
|
|
769
787
|
```bash
|
|
770
|
-
python -m unittest discover -s tests -p "test_*.py" -v #
|
|
788
|
+
python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
|
|
771
789
|
python -m ruff check . # lint
|
|
772
790
|
python -m mypy xssmap # types on shipped package
|
|
773
791
|
```
|
|
@@ -867,7 +885,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
|
|
|
867
885
|
by at most the remaining slice (floored at 0.25s), and the report is then marked
|
|
868
886
|
`partial` rather than pretending the sweep finished.
|
|
869
887
|
- Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
|
|
870
|
-
`--threads`; the scanner honours those limits for every request it makes.
|
|
888
|
+
`--crawl-threads`; the scanner honours those limits for every request it makes.
|
|
889
|
+
A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
|
|
890
|
+
is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
|
|
871
891
|
- WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
|
|
872
892
|
makes a payload reflect verbatim is tried first on later points, and one that only
|
|
873
893
|
produces mangled or blocked responses moves to the end of the budget (the per-scan
|
|
@@ -487,6 +487,23 @@ request. Three things make the result usable rather than noisy:
|
|
|
487
487
|
A flagged version is a lead: it says the build has a published problem, not that this
|
|
488
488
|
host is exploitable through it. The finding standard for XSS is unchanged.
|
|
489
489
|
|
|
490
|
+
### Crawl and scan speed
|
|
491
|
+
|
|
492
|
+
Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
|
|
493
|
+
cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
|
|
494
|
+
pages with 50 ms of latency per response, the same code, same page set and the same
|
|
495
|
+
8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
|
|
496
|
+
the current ones** (16 discovery workers, 10 payload workers). At a real WAN
|
|
497
|
+
latency of 200-300 ms the difference is the same multiple, in minutes.
|
|
498
|
+
|
|
499
|
+
Discovery therefore has its own worker count. It only fetches pages the site
|
|
500
|
+
links to itself, and the shared throttle still backs off on 429/503, so it does
|
|
501
|
+
not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
|
|
502
|
+
`--threads` remains the knob for the scan phase. Two scan-side savings are
|
|
503
|
+
always on: the baseline response is fetched once per URL instead of once per
|
|
504
|
+
parameter, and the point order is deterministic, so `--max-points` cuts the same
|
|
505
|
+
places on every run.
|
|
506
|
+
|
|
490
507
|
### Browser cost
|
|
491
508
|
|
|
492
509
|
`--browser` reuses one browser and one page per run. The report's `browser_verification`
|
|
@@ -547,7 +564,8 @@ per check, so the price of execution proof is visible rather than implied.
|
|
|
547
564
|
| `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
|
|
548
565
|
| `--headless new\|old` | headless mode passed to the browser |
|
|
549
566
|
| `--screenshots` | capture a PNG for verified findings |
|
|
550
|
-
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
|
|
567
|
+
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
|
|
568
|
+
| `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
|
|
551
569
|
| `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
|
|
552
570
|
| `--max-points N` | stop after N injection points |
|
|
553
571
|
| `--state FILE` | resume file: skip points already completed, carry findings over |
|
|
@@ -743,7 +761,7 @@ response shape shows up as a miss on the other.
|
|
|
743
761
|
## Tests and CI
|
|
744
762
|
|
|
745
763
|
```bash
|
|
746
|
-
python -m unittest discover -s tests -p "test_*.py" -v #
|
|
764
|
+
python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
|
|
747
765
|
python -m ruff check . # lint
|
|
748
766
|
python -m mypy xssmap # types on shipped package
|
|
749
767
|
```
|
|
@@ -843,7 +861,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
|
|
|
843
861
|
by at most the remaining slice (floored at 0.25s), and the report is then marked
|
|
844
862
|
`partial` rather than pretending the sweep finished.
|
|
845
863
|
- Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
|
|
846
|
-
`--threads`; the scanner honours those limits for every request it makes.
|
|
864
|
+
`--crawl-threads`; the scanner honours those limits for every request it makes.
|
|
865
|
+
A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
|
|
866
|
+
is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
|
|
847
867
|
- WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
|
|
848
868
|
makes a payload reflect verbatim is tried first on later points, and one that only
|
|
849
869
|
produces mangled or blocked responses moves to the end of the budget (the per-scan
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Tests for the request-count and worker-count rules behind scan speed.
|
|
2
|
+
|
|
3
|
+
Two things make a 4,000-URL scan slow, and both are counted here rather than
|
|
4
|
+
timed (timing tests are flaky in CI):
|
|
5
|
+
|
|
6
|
+
* discovery ran at ``--threads`` (5) workers, so a 4,096-page crawl was 800
|
|
7
|
+
sequential round trips. Discovery now has its own, wider default.
|
|
8
|
+
* every parameter of a URL paid for its own baseline request even though the
|
|
9
|
+
unmodified request is byte-identical. The baseline is shared now, and only
|
|
10
|
+
its status is kept, so nothing large is cached.
|
|
11
|
+
|
|
12
|
+
Run: python -m unittest tests.test_speed -v
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
import unittest
|
|
19
|
+
|
|
20
|
+
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
21
|
+
sys.path.insert(0, ROOT)
|
|
22
|
+
|
|
23
|
+
from xssmap import cli, scanner # noqa: E402
|
|
24
|
+
from xssmap.net import Response # noqa: E402
|
|
25
|
+
|
|
26
|
+
FLAT = "<html><body><p>nothing is echoed here</p></body></html>"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _CountingHTTP:
|
|
30
|
+
"""Answers with a body that never contains the probe; counts every request."""
|
|
31
|
+
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self.requests = []
|
|
34
|
+
|
|
35
|
+
def send(self, req, allow_redirects: bool = False):
|
|
36
|
+
self.requests.append(req)
|
|
37
|
+
return Response(url=req.url, method=req.method, status=200,
|
|
38
|
+
headers=[("content-type", "text/html; charset=utf-8")],
|
|
39
|
+
body=FLAT.encode("utf-8"), elapsed=0.0)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _points(url: str) -> list:
|
|
43
|
+
return scanner.points_from_url(url, [])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class BaselineSharingTest(unittest.TestCase):
|
|
47
|
+
def _run(self, points, concurrency=1):
|
|
48
|
+
http = _CountingHTTP()
|
|
49
|
+
scan = scanner.Scanner(http=http, browser=None)
|
|
50
|
+
findings = scan.scan_points(points, concurrency=concurrency)
|
|
51
|
+
return http, findings
|
|
52
|
+
|
|
53
|
+
def test_second_parameter_reuses_the_first_baseline(self):
|
|
54
|
+
# ?a=1&b=2 -> two points; their baseline request is the same URL, so the
|
|
55
|
+
# second point must not pay for it again.
|
|
56
|
+
http, findings = self._run(_points("https://t.example/search?a=1&b=2"))
|
|
57
|
+
self.assertEqual(findings, [])
|
|
58
|
+
self.assertEqual(len(http.requests), 5,
|
|
59
|
+
"two points x 3 requests minus one shared baseline")
|
|
60
|
+
|
|
61
|
+
def test_parallel_workers_do_not_fetch_the_same_baseline_twice(self):
|
|
62
|
+
# Both points start before either baseline answered; the second worker
|
|
63
|
+
# must wait for the first one's answer instead of re-fetching the page.
|
|
64
|
+
http, findings = self._run(_points("https://t.example/search?a=1&b=2"),
|
|
65
|
+
concurrency=2)
|
|
66
|
+
self.assertEqual(findings, [])
|
|
67
|
+
self.assertEqual(len(http.requests), 5)
|
|
68
|
+
|
|
69
|
+
def test_one_parameter_still_costs_its_own_baseline(self):
|
|
70
|
+
http, _ = self._run(_points("https://t.example/search?a=1"))
|
|
71
|
+
self.assertEqual(len(http.requests), 3)
|
|
72
|
+
|
|
73
|
+
def test_different_urls_do_not_share_a_baseline(self):
|
|
74
|
+
# Same parameter names, different path: the baseline is a different
|
|
75
|
+
# request, so both points must send one.
|
|
76
|
+
http, _ = self._run(_points("https://t.example/a?q=1") + _points("https://t.example/b?q=1"))
|
|
77
|
+
self.assertEqual(len(http.requests), 6)
|
|
78
|
+
|
|
79
|
+
def test_cache_only_holds_statuses(self):
|
|
80
|
+
http = _CountingHTTP()
|
|
81
|
+
scan = scanner.Scanner(http=http, browser=None)
|
|
82
|
+
scan.scan_points(_points("https://t.example/search?a=1&b=2"), concurrency=1)
|
|
83
|
+
self.assertTrue(all(isinstance(v, int) for v in scan._baseline_status_cache.values()),
|
|
84
|
+
"response bodies must not be kept alive by the cache")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class CrawlWorkerTest(unittest.TestCase):
|
|
88
|
+
def _args(self, argv):
|
|
89
|
+
parser = cli.build_parser()
|
|
90
|
+
args = parser.parse_args(argv)
|
|
91
|
+
cli.finalize_args(args)
|
|
92
|
+
return args
|
|
93
|
+
|
|
94
|
+
def test_discovery_runs_wider_than_payload_testing_by_default(self):
|
|
95
|
+
args = self._args(["-u", "https://t.example/"])
|
|
96
|
+
self.assertEqual(args.threads, 10)
|
|
97
|
+
self.assertEqual(args.crawl_threads, 16)
|
|
98
|
+
|
|
99
|
+
def test_explicit_crawl_threads_wins(self):
|
|
100
|
+
args = self._args(["-u", "https://t.example/", "--crawl-threads", "4"])
|
|
101
|
+
self.assertEqual(args.crawl_threads, 4)
|
|
102
|
+
|
|
103
|
+
def test_payload_threads_raise_the_discovery_floor(self):
|
|
104
|
+
# Asking for 40 payload workers must not leave discovery at 16.
|
|
105
|
+
args = self._args(["-u", "https://t.example/", "--threads", "40"])
|
|
106
|
+
self.assertEqual(args.crawl_threads, 40)
|
|
107
|
+
|
|
108
|
+
def test_autonomous_keeps_the_wider_discovery_default(self):
|
|
109
|
+
args = self._args(["--domain", "example.com", "--autonomous"])
|
|
110
|
+
self.assertGreaterEqual(args.crawl_threads, 16)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
unittest.main()
|
|
@@ -40,6 +40,9 @@ from .scanner import (
|
|
|
40
40
|
from .session import CookieJar, Session, is_login_form
|
|
41
41
|
from .stored import StoredScanner
|
|
42
42
|
|
|
43
|
+
# Discovery workers used when the user does not pass --crawl-threads.
|
|
44
|
+
DEFAULT_CRAWL_THREADS = 16
|
|
45
|
+
|
|
43
46
|
EPILOG = """examples:
|
|
44
47
|
xssmap -u "https://target.tld/search?q=test"
|
|
45
48
|
xssmap -u https://target.tld --crawl --depth 3 --max-pages 200
|
|
@@ -148,7 +151,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
148
151
|
help="never submit POST forms (overrides --autonomous)")
|
|
149
152
|
scan.add_argument("--headers-points", action="store_true",
|
|
150
153
|
help="test User-Agent/Referer/X-Forwarded-Host reflection")
|
|
151
|
-
scan.add_argument("--threads", type=int, default=
|
|
154
|
+
scan.add_argument("--threads", type=int, default=10,
|
|
155
|
+
help="payload-testing workers (default 10)")
|
|
156
|
+
scan.add_argument("--crawl-threads", type=int, default=0, metavar="N",
|
|
157
|
+
help="discovery workers for the crawl (default: 16, or --threads "
|
|
158
|
+
"if larger)")
|
|
152
159
|
scan.add_argument("--delay", type=float, default=0.0, help="minimum delay between requests")
|
|
153
160
|
scan.add_argument("--timeout", type=float, default=15.0, help="per-request timeout")
|
|
154
161
|
scan.add_argument("--retries", type=int, default=2, help="retries on transport errors")
|
|
@@ -390,6 +397,12 @@ def finalize_args(args: argparse.Namespace) -> None:
|
|
|
390
397
|
args.crawl = False
|
|
391
398
|
elif args.crawl is None:
|
|
392
399
|
args.crawl = True
|
|
400
|
+
if getattr(args, "crawl_threads", 0) <= 0:
|
|
401
|
+
# Discovery only fetches the site's own links, and the shared throttle
|
|
402
|
+
# already backs off on 429/503, so it does not need to run at the same
|
|
403
|
+
# narrow width as payload testing. Measured on a 4096-page lab target:
|
|
404
|
+
# 5 workers 46 s, 16 workers 15 s, 32 workers 7 s.
|
|
405
|
+
args.crawl_threads = max(DEFAULT_CRAWL_THREADS, args.threads)
|
|
393
406
|
|
|
394
407
|
|
|
395
408
|
def read_wordlist(path: str) -> list[str]:
|
|
@@ -794,11 +807,11 @@ def main(argv=None) -> int:
|
|
|
794
807
|
if do_crawl:
|
|
795
808
|
hosts_note = f", {len(seeds)} host(s)" if len(seeds) > 1 else ""
|
|
796
809
|
log.action(f"crawling (depth {args.crawl_depth}, max {args.max_pages} pages, "
|
|
797
|
-
f"{args.
|
|
810
|
+
f"{args.crawl_threads} workers{hosts_note})")
|
|
798
811
|
crawler = Crawler(
|
|
799
812
|
http, scope, log,
|
|
800
813
|
max_depth=args.crawl_depth, max_pages=args.max_pages,
|
|
801
|
-
concurrency=args.
|
|
814
|
+
concurrency=args.crawl_threads, exclude=args.exclude, include=args.include,
|
|
802
815
|
crawl_js=not args.no_js, crawl_forms=True, ajax=not args.no_js,
|
|
803
816
|
respect_robots=args.robots, headers=headers,
|
|
804
817
|
)
|
|
@@ -489,8 +489,12 @@ class Crawler:
|
|
|
489
489
|
self.result.param_urls.append(url)
|
|
490
490
|
|
|
491
491
|
def _finish_param_urls(self):
|
|
492
|
-
#
|
|
493
|
-
|
|
492
|
+
# Shallow/short URLs get scanned first. The URL itself is the final key:
|
|
493
|
+
# without it, ties were broken by crawl completion order, so the same
|
|
494
|
+
# target produced a different point order (and a different --max-points
|
|
495
|
+
# cut) on every run.
|
|
496
|
+
self.result.param_urls.sort(
|
|
497
|
+
key=lambda u: (len(urllib.parse.urlsplit(u).query), len(u), u))
|
|
494
498
|
|
|
495
499
|
|
|
496
500
|
def _title_of(text: str) -> str:
|
|
@@ -394,6 +394,11 @@ class Scanner:
|
|
|
394
394
|
self.skip_points: set[str] = set(skip_points or ())
|
|
395
395
|
self.completed_points: set[str] = set()
|
|
396
396
|
self.resumed_points = 0
|
|
397
|
+
self._baseline_status_cache: dict[str, int] = {}
|
|
398
|
+
# One in-flight baseline per request: two parameters of the same URL are
|
|
399
|
+
# queued next to each other, so without this the workers both miss the
|
|
400
|
+
# cache and both fetch the same page.
|
|
401
|
+
self._baseline_inflight: dict[str, threading.Event] = {}
|
|
397
402
|
self.profile_enabled = profile_enabled
|
|
398
403
|
self.mutation_budget = max(0, mutation_budget)
|
|
399
404
|
self.interrupted = False
|
|
@@ -544,6 +549,45 @@ class Scanner:
|
|
|
544
549
|
self.completed_points.add(key)
|
|
545
550
|
return found
|
|
546
551
|
|
|
552
|
+
def _baseline_status(self, req: Request) -> int | None:
|
|
553
|
+
"""Status of a baseline request, cached per method+URL+body.
|
|
554
|
+
|
|
555
|
+
A URL with three parameters produces three points whose baseline request
|
|
556
|
+
is byte-identical, and the engine only reads ``.status`` from it. Bodies
|
|
557
|
+
are never cached, so this costs a dictionary of integers rather than a
|
|
558
|
+
copy of every page.
|
|
559
|
+
"""
|
|
560
|
+
key = f"{req.method} {req.url}\x00{req.body!r}"
|
|
561
|
+
event: threading.Event
|
|
562
|
+
with self.lock:
|
|
563
|
+
cached = self._baseline_status_cache.get(key)
|
|
564
|
+
if cached is not None:
|
|
565
|
+
return cached
|
|
566
|
+
existing = self._baseline_inflight.get(key)
|
|
567
|
+
owner = existing is None
|
|
568
|
+
if existing is None:
|
|
569
|
+
event = threading.Event()
|
|
570
|
+
self._baseline_inflight[key] = event
|
|
571
|
+
else:
|
|
572
|
+
event = existing
|
|
573
|
+
if not owner:
|
|
574
|
+
# Another worker is fetching exactly this baseline; its answer is ours.
|
|
575
|
+
event.wait(getattr(self.http, "timeout", 15.0) + 5.0)
|
|
576
|
+
with self.lock:
|
|
577
|
+
return self._baseline_status_cache.get(key)
|
|
578
|
+
try:
|
|
579
|
+
status: int | None = self._send(req).status
|
|
580
|
+
except NetError:
|
|
581
|
+
status = None
|
|
582
|
+
with self.lock:
|
|
583
|
+
if status is not None:
|
|
584
|
+
if len(self._baseline_status_cache) >= 4096:
|
|
585
|
+
self._baseline_status_cache.clear()
|
|
586
|
+
self._baseline_status_cache[key] = status
|
|
587
|
+
self._baseline_inflight.pop(key, None)
|
|
588
|
+
event.set()
|
|
589
|
+
return status
|
|
590
|
+
|
|
547
591
|
def _point_key(self, p: InjectionPoint) -> str:
|
|
548
592
|
if isinstance(p, QueryPoint):
|
|
549
593
|
return f"q:{p.url.split('#')[0]}:{p.index}"
|
|
@@ -573,21 +617,24 @@ class Scanner:
|
|
|
573
617
|
else:
|
|
574
618
|
value_original = original
|
|
575
619
|
|
|
576
|
-
# 1. baseline with the original value (some apps 500 on empty params)
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
620
|
+
# 1. baseline with the original value (some apps 500 on empty params).
|
|
621
|
+
# Only the status is ever consulted, so it is shared between points that
|
|
622
|
+
# build the identical request -- the second parameter of a URL does not
|
|
623
|
+
# need its own copy of the same round trip.
|
|
624
|
+
baseline_status = self._baseline_status(point.build(value_original))
|
|
581
625
|
|
|
582
626
|
marker = new_marker()
|
|
583
627
|
marker_value = f"{marker}zqx"
|
|
584
628
|
resp = self._send(point.build(marker_value))
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
629
|
+
# A 500 here where the unmodified URL answered fine means our value broke
|
|
630
|
+
# the request. The old code answered that by re-sending the identical
|
|
631
|
+
# probe, which can only reproduce the 500 and costs a round trip; the
|
|
632
|
+
# error body is analysed below instead, because some applications echo
|
|
633
|
+
# the value inside their error page.
|
|
634
|
+
if (self.log and baseline_status is not None and resp.status >= 500
|
|
635
|
+
and baseline_status < 500):
|
|
636
|
+
self.log.debug(f" {point.label()} answers {resp.status} with our value "
|
|
637
|
+
f"({baseline_status} unmodified); error body kept")
|
|
591
638
|
|
|
592
639
|
marker, resp, match = self._settle_probe(point, marker, resp)
|
|
593
640
|
text = resp.text
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xssmap
|
|
3
|
-
Version: 2.9.
|
|
3
|
+
Version: 2.9.1
|
|
4
4
|
Summary: Context-aware XSS scanner: crawler, browser verification, mXSS, charset and stored-XSS stages
|
|
5
5
|
Author: Ian Mattas
|
|
6
6
|
License-Expression: MIT
|
|
@@ -511,6 +511,23 @@ request. Three things make the result usable rather than noisy:
|
|
|
511
511
|
A flagged version is a lead: it says the build has a published problem, not that this
|
|
512
512
|
host is exploitable through it. The finding standard for XSS is unchanged.
|
|
513
513
|
|
|
514
|
+
### Crawl and scan speed
|
|
515
|
+
|
|
516
|
+
Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
|
|
517
|
+
cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
|
|
518
|
+
pages with 50 ms of latency per response, the same code, same page set and the same
|
|
519
|
+
8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
|
|
520
|
+
the current ones** (16 discovery workers, 10 payload workers). At a real WAN
|
|
521
|
+
latency of 200-300 ms the difference is the same multiple, in minutes.
|
|
522
|
+
|
|
523
|
+
Discovery therefore has its own worker count. It only fetches pages the site
|
|
524
|
+
links to itself, and the shared throttle still backs off on 429/503, so it does
|
|
525
|
+
not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
|
|
526
|
+
`--threads` remains the knob for the scan phase. Two scan-side savings are
|
|
527
|
+
always on: the baseline response is fetched once per URL instead of once per
|
|
528
|
+
parameter, and the point order is deterministic, so `--max-points` cuts the same
|
|
529
|
+
places on every run.
|
|
530
|
+
|
|
514
531
|
### Browser cost
|
|
515
532
|
|
|
516
533
|
`--browser` reuses one browser and one page per run. The report's `browser_verification`
|
|
@@ -571,7 +588,8 @@ per check, so the price of execution proof is visible rather than implied.
|
|
|
571
588
|
| `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
|
|
572
589
|
| `--headless new\|old` | headless mode passed to the browser |
|
|
573
590
|
| `--screenshots` | capture a PNG for verified findings |
|
|
574
|
-
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
|
|
591
|
+
| `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
|
|
592
|
+
| `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
|
|
575
593
|
| `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
|
|
576
594
|
| `--max-points N` | stop after N injection points |
|
|
577
595
|
| `--state FILE` | resume file: skip points already completed, carry findings over |
|
|
@@ -767,7 +785,7 @@ response shape shows up as a miss on the other.
|
|
|
767
785
|
## Tests and CI
|
|
768
786
|
|
|
769
787
|
```bash
|
|
770
|
-
python -m unittest discover -s tests -p "test_*.py" -v #
|
|
788
|
+
python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
|
|
771
789
|
python -m ruff check . # lint
|
|
772
790
|
python -m mypy xssmap # types on shipped package
|
|
773
791
|
```
|
|
@@ -867,7 +885,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
|
|
|
867
885
|
by at most the remaining slice (floored at 0.25s), and the report is then marked
|
|
868
886
|
`partial` rather than pretending the sweep finished.
|
|
869
887
|
- Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
|
|
870
|
-
`--threads`; the scanner honours those limits for every request it makes.
|
|
888
|
+
`--crawl-threads`; the scanner honours those limits for every request it makes.
|
|
889
|
+
A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
|
|
890
|
+
is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
|
|
871
891
|
- WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
|
|
872
892
|
makes a payload reflect verbatim is tried first on later points, and one that only
|
|
873
893
|
produces mangled or blocked responses moves to the end of the budget (the per-scan
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|