xssmap 2.9.0__tar.gz → 2.9.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {xssmap-2.9.0 → xssmap-2.9.1}/CHANGELOG.md +28 -4
  2. {xssmap-2.9.0/xssmap.egg-info → xssmap-2.9.1}/PKG-INFO +24 -4
  3. {xssmap-2.9.0 → xssmap-2.9.1}/README.md +23 -3
  4. xssmap-2.9.1/tests/test_speed.py +114 -0
  5. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/__init__.py +1 -1
  6. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/cli.py +16 -3
  7. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/crawler.py +6 -2
  8. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/scanner.py +58 -11
  9. {xssmap-2.9.0 → xssmap-2.9.1/xssmap.egg-info}/PKG-INFO +24 -4
  10. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/SOURCES.txt +1 -0
  11. {xssmap-2.9.0 → xssmap-2.9.1}/LICENSE +0 -0
  12. {xssmap-2.9.0 → xssmap-2.9.1}/MANIFEST.in +0 -0
  13. {xssmap-2.9.0 → xssmap-2.9.1}/pyproject.toml +0 -0
  14. {xssmap-2.9.0 → xssmap-2.9.1}/setup.cfg +0 -0
  15. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_bench.py +0 -0
  16. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_brute.py +0 -0
  17. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_budget.py +0 -0
  18. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_charset.py +0 -0
  19. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_crawlperf.py +0 -0
  20. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_csp.py +0 -0
  21. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_domscan.py +0 -0
  22. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_domstate.py +0 -0
  23. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_efficiency.py +0 -0
  24. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_exploit.py +0 -0
  25. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_fuzz.py +0 -0
  26. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_honesty.py +0 -0
  27. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_inline.py +0 -0
  28. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_jslib.py +0 -0
  29. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_leads.py +0 -0
  30. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_mxss.py +0 -0
  31. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_ops.py +0 -0
  32. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_params.py +0 -0
  33. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_recon_report.py +0 -0
  34. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_session.py +0 -0
  35. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_subdomains.py +0 -0
  36. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_tech.py +0 -0
  37. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_units.py +0 -0
  38. {xssmap-2.9.0 → xssmap-2.9.1}/tests/test_waf.py +0 -0
  39. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/__main__.py +0 -0
  40. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/browser.py +0 -0
  41. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/brute.py +0 -0
  42. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/callback.py +0 -0
  43. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/charset.py +0 -0
  44. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/contexts.py +0 -0
  45. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/csp.py +0 -0
  46. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/domscan.py +0 -0
  47. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/domstate.py +0 -0
  48. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/efficiency.py +0 -0
  49. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/exploit.py +0 -0
  50. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/filters.py +0 -0
  51. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/fuzz.py +0 -0
  52. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/htmlparse.py +0 -0
  53. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/jslib.py +0 -0
  54. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/mxss.py +0 -0
  55. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/net.py +0 -0
  56. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/params.py +0 -0
  57. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/payloads.py +0 -0
  58. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/py.typed +0 -0
  59. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/report.py +0 -0
  60. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/repro.py +0 -0
  61. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/sarif.py +0 -0
  62. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/session.py +0 -0
  63. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/state.py +0 -0
  64. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/stored.py +0 -0
  65. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/subdomains.py +0 -0
  66. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/tech.py +0 -0
  67. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/verify.py +0 -0
  68. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap/waf.py +0 -0
  69. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/dependency_links.txt +0 -0
  70. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/entry_points.txt +0 -0
  71. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/requires.txt +0 -0
  72. {xssmap-2.9.0 → xssmap-2.9.1}/xssmap.egg-info/top_level.txt +0 -0
@@ -8,6 +8,29 @@ Releases before 2.8.0 predate this file; their history is in the git log.
8
8
 
9
9
  ## [Unreleased]
10
10
 
11
+ ## [2.9.1] - 2026-11-13
12
+
13
+ Speed release. Detection behaviour is unchanged.
14
+
15
+ ### Changed
16
+
17
+ - Discovery has its own worker count. `--crawl-threads` defaults to 16 (or
18
+ `--threads` when that is larger) instead of running the crawl at the payload
19
+ width of 5; `--threads` itself now defaults to 10. On a 4,096-page lab target
20
+ with 50 ms of latency, the same 8,252 requests finished in 85 s before and
21
+ 38 s after.
22
+ - The baseline response is fetched once per URL rather than once per parameter,
23
+ and concurrent workers share the in-flight fetch instead of racing on it.
24
+ Only the status is cached, never a body.
25
+
26
+ ### Fixed
27
+
28
+ - Point order is deterministic. `param_urls` ties were broken by crawl
29
+ completion order, so `--max-points` could cut at a different place on every
30
+ run of the same target.
31
+ - A parameter that answers 500 when probed no longer re-sends the identical
32
+ probe; the error body is still analysed for a reflection.
33
+
11
34
  ## [2.9.0] - 2026-11-13
12
35
 
13
36
  ### Added
@@ -23,7 +46,7 @@ Releases before 2.8.0 predate this file; their history is in the git log.
23
46
  8.2 (out of security support) and IIS 6.0-8.5 (shipped only with an
24
47
  end-of-life Windows Server). A flagged version is a lead for the operator,
25
48
  never a finding on its own.
26
- - 35 tests pinning the new crawl scheduling, the transport cap, the throttle
49
+ - 36 tests pinning the new crawl scheduling, the transport cap, the throttle
27
50
  rules and the stack policy table.
28
51
 
29
52
  ### Changed
@@ -43,9 +66,10 @@ Releases before 2.8.0 predate this file; their history is in the git log.
43
66
 
44
67
  ### Fixed
45
68
 
46
- - Crawling a target that 500s part of its URL space is ~24x faster at the same
47
- page count: 18.90 s to 0.79 s for 68 requests in the bundled benchmark
48
- (identical page sets, deterministic error injection).
69
+ - Crawling a target that 500s part of its URL space is ~20x faster at the same
70
+ page count: about 1 s instead of 20 s for 68 pages, measured by running the
71
+ 2.8.1 tree and this one against the same local server (identical page sets,
72
+ 50 ms of fixed latency, a deterministic 15% of URLs answering 500).
49
73
 
50
74
  ## [2.8.1] - 2026-11-13
51
75
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xssmap
3
- Version: 2.9.0
3
+ Version: 2.9.1
4
4
  Summary: Context-aware XSS scanner: crawler, browser verification, mXSS, charset and stored-XSS stages
5
5
  Author: Ian Mattas
6
6
  License-Expression: MIT
@@ -511,6 +511,23 @@ request. Three things make the result usable rather than noisy:
511
511
  A flagged version is a lead: it says the build has a published problem, not that this
512
512
  host is exploitable through it. The finding standard for XSS is unchanged.
513
513
 
514
+ ### Crawl and scan speed
515
+
516
+ Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
517
+ cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
518
+ pages with 50 ms of latency per response, the same code, same page set and the same
519
+ 8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
520
+ the current ones** (16 discovery workers, 10 payload workers). At a real WAN
521
+ latency of 200-300 ms the difference is the same multiple, in minutes.
522
+
523
+ Discovery therefore has its own worker count. It only fetches pages the site
524
+ links to itself, and the shared throttle still backs off on 429/503, so it does
525
+ not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
526
+ `--threads` remains the knob for the scan phase. Two scan-side savings are
527
+ always on: the baseline response is fetched once per URL instead of once per
528
+ parameter, and the point order is deterministic, so `--max-points` cuts the same
529
+ places on every run.
530
+
514
531
  ### Browser cost
515
532
 
516
533
  `--browser` reuses one browser and one page per run. The report's `browser_verification`
@@ -571,7 +588,8 @@ per check, so the price of execution proof is visible rather than implied.
571
588
  | `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
572
589
  | `--headless new\|old` | headless mode passed to the browser |
573
590
  | `--screenshots` | capture a PNG for verified findings |
574
- | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
591
+ | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
592
+ | `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
575
593
  | `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
576
594
  | `--max-points N` | stop after N injection points |
577
595
  | `--state FILE` | resume file: skip points already completed, carry findings over |
@@ -767,7 +785,7 @@ response shape shows up as a miss on the other.
767
785
  ## Tests and CI
768
786
 
769
787
  ```bash
770
- python -m unittest discover -s tests -p "test_*.py" -v # 445 tests, no network
788
+ python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
771
789
  python -m ruff check . # lint
772
790
  python -m mypy xssmap # types on shipped package
773
791
  ```
@@ -867,7 +885,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
867
885
  by at most the remaining slice (floored at 0.25s), and the report is then marked
868
886
  `partial` rather than pretending the sweep finished.
869
887
  - Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
870
- `--threads`; the scanner honours those limits for every request it makes.
888
+ `--crawl-threads`; the scanner honours those limits for every request it makes.
889
+ A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
890
+ is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
871
891
  - WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
872
892
  makes a payload reflect verbatim is tried first on later points, and one that only
873
893
  produces mangled or blocked responses moves to the end of the budget (the per-scan
@@ -487,6 +487,23 @@ request. Three things make the result usable rather than noisy:
487
487
  A flagged version is a lead: it says the build has a published problem, not that this
488
488
  host is exploitable through it. The finding standard for XSS is unchanged.
489
489
 
490
+ ### Crawl and scan speed
491
+
492
+ Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
493
+ cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
494
+ pages with 50 ms of latency per response, the same code, same page set and the same
495
+ 8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
496
+ the current ones** (16 discovery workers, 10 payload workers). At a real WAN
497
+ latency of 200-300 ms the difference is the same multiple, in minutes.
498
+
499
+ Discovery therefore has its own worker count. It only fetches pages the site
500
+ links to itself, and the shared throttle still backs off on 429/503, so it does
501
+ not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
502
+ `--threads` remains the knob for the scan phase. Two scan-side savings are
503
+ always on: the baseline response is fetched once per URL instead of once per
504
+ parameter, and the point order is deterministic, so `--max-points` cuts the same
505
+ places on every run.
506
+
490
507
  ### Browser cost
491
508
 
492
509
  `--browser` reuses one browser and one page per run. The report's `browser_verification`
@@ -547,7 +564,8 @@ per check, so the price of execution proof is visible rather than implied.
547
564
  | `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
548
565
  | `--headless new\|old` | headless mode passed to the browser |
549
566
  | `--screenshots` | capture a PNG for verified findings |
550
- | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
567
+ | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
568
+ | `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
551
569
  | `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
552
570
  | `--max-points N` | stop after N injection points |
553
571
  | `--state FILE` | resume file: skip points already completed, carry findings over |
@@ -743,7 +761,7 @@ response shape shows up as a miss on the other.
743
761
  ## Tests and CI
744
762
 
745
763
  ```bash
746
- python -m unittest discover -s tests -p "test_*.py" -v # 445 tests, no network
764
+ python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
747
765
  python -m ruff check . # lint
748
766
  python -m mypy xssmap # types on shipped package
749
767
  ```
@@ -843,7 +861,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
843
861
  by at most the remaining slice (floored at 0.25s), and the report is then marked
844
862
  `partial` rather than pretending the sweep finished.
845
863
  - Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
846
- `--threads`; the scanner honours those limits for every request it makes.
864
+ `--crawl-threads`; the scanner honours those limits for every request it makes.
865
+ A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
866
+ is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
847
867
  - WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
848
868
  makes a payload reflect verbatim is tried first on later points, and one that only
849
869
  produces mangled or blocked responses moves to the end of the budget (the per-scan
@@ -0,0 +1,114 @@
1
+ """Tests for the request-count and worker-count rules behind scan speed.
2
+
3
+ Two things make a 4,000-URL scan slow, and both are counted here rather than
4
+ timed (timing tests are flaky in CI):
5
+
6
+ * discovery ran at ``--threads`` (5) workers, so a 4,096-page crawl was 800
7
+ sequential round trips. Discovery now has its own, wider default.
8
+ * every parameter of a URL paid for its own baseline request even though the
9
+ unmodified request is byte-identical. The baseline is shared now, and only
10
+ its status is kept, so nothing large is cached.
11
+
12
+ Run: python -m unittest tests.test_speed -v
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import sys
18
+ import unittest
19
+
20
+ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
+ sys.path.insert(0, ROOT)
22
+
23
+ from xssmap import cli, scanner # noqa: E402
24
+ from xssmap.net import Response # noqa: E402
25
+
26
+ FLAT = "<html><body><p>nothing is echoed here</p></body></html>"
27
+
28
+
29
+ class _CountingHTTP:
30
+ """Answers with a body that never contains the probe; counts every request."""
31
+
32
+ def __init__(self):
33
+ self.requests = []
34
+
35
+ def send(self, req, allow_redirects: bool = False):
36
+ self.requests.append(req)
37
+ return Response(url=req.url, method=req.method, status=200,
38
+ headers=[("content-type", "text/html; charset=utf-8")],
39
+ body=FLAT.encode("utf-8"), elapsed=0.0)
40
+
41
+
42
+ def _points(url: str) -> list:
43
+ return scanner.points_from_url(url, [])
44
+
45
+
46
+ class BaselineSharingTest(unittest.TestCase):
47
+ def _run(self, points, concurrency=1):
48
+ http = _CountingHTTP()
49
+ scan = scanner.Scanner(http=http, browser=None)
50
+ findings = scan.scan_points(points, concurrency=concurrency)
51
+ return http, findings
52
+
53
+ def test_second_parameter_reuses_the_first_baseline(self):
54
+ # ?a=1&b=2 -> two points; their baseline request is the same URL, so the
55
+ # second point must not pay for it again.
56
+ http, findings = self._run(_points("https://t.example/search?a=1&b=2"))
57
+ self.assertEqual(findings, [])
58
+ self.assertEqual(len(http.requests), 5,
59
+ "two points x 3 requests minus one shared baseline")
60
+
61
+ def test_parallel_workers_do_not_fetch_the_same_baseline_twice(self):
62
+ # Both points start before either baseline answered; the second worker
63
+ # must wait for the first one's answer instead of re-fetching the page.
64
+ http, findings = self._run(_points("https://t.example/search?a=1&b=2"),
65
+ concurrency=2)
66
+ self.assertEqual(findings, [])
67
+ self.assertEqual(len(http.requests), 5)
68
+
69
+ def test_one_parameter_still_costs_its_own_baseline(self):
70
+ http, _ = self._run(_points("https://t.example/search?a=1"))
71
+ self.assertEqual(len(http.requests), 3)
72
+
73
+ def test_different_urls_do_not_share_a_baseline(self):
74
+ # Same parameter names, different path: the baseline is a different
75
+ # request, so both points must send one.
76
+ http, _ = self._run(_points("https://t.example/a?q=1") + _points("https://t.example/b?q=1"))
77
+ self.assertEqual(len(http.requests), 6)
78
+
79
+ def test_cache_only_holds_statuses(self):
80
+ http = _CountingHTTP()
81
+ scan = scanner.Scanner(http=http, browser=None)
82
+ scan.scan_points(_points("https://t.example/search?a=1&b=2"), concurrency=1)
83
+ self.assertTrue(all(isinstance(v, int) for v in scan._baseline_status_cache.values()),
84
+ "response bodies must not be kept alive by the cache")
85
+
86
+
87
+ class CrawlWorkerTest(unittest.TestCase):
88
+ def _args(self, argv):
89
+ parser = cli.build_parser()
90
+ args = parser.parse_args(argv)
91
+ cli.finalize_args(args)
92
+ return args
93
+
94
+ def test_discovery_runs_wider_than_payload_testing_by_default(self):
95
+ args = self._args(["-u", "https://t.example/"])
96
+ self.assertEqual(args.threads, 10)
97
+ self.assertEqual(args.crawl_threads, 16)
98
+
99
+ def test_explicit_crawl_threads_wins(self):
100
+ args = self._args(["-u", "https://t.example/", "--crawl-threads", "4"])
101
+ self.assertEqual(args.crawl_threads, 4)
102
+
103
+ def test_payload_threads_raise_the_discovery_floor(self):
104
+ # Asking for 40 payload workers must not leave discovery at 16.
105
+ args = self._args(["-u", "https://t.example/", "--threads", "40"])
106
+ self.assertEqual(args.crawl_threads, 40)
107
+
108
+ def test_autonomous_keeps_the_wider_discovery_default(self):
109
+ args = self._args(["--domain", "example.com", "--autonomous"])
110
+ self.assertGreaterEqual(args.crawl_threads, 16)
111
+
112
+
113
+ if __name__ == "__main__":
114
+ unittest.main()
@@ -1,6 +1,6 @@
1
1
  """xssmap - context-aware cross-site scripting scanner with a crawler."""
2
2
 
3
- __version__ = "2.9.0"
3
+ __version__ = "2.9.1"
4
4
 
5
5
  __all__ = ["__version__", "main"]
6
6
 
@@ -40,6 +40,9 @@ from .scanner import (
40
40
  from .session import CookieJar, Session, is_login_form
41
41
  from .stored import StoredScanner
42
42
 
43
+ # Discovery workers used when the user does not pass --crawl-threads.
44
+ DEFAULT_CRAWL_THREADS = 16
45
+
43
46
  EPILOG = """examples:
44
47
  xssmap -u "https://target.tld/search?q=test"
45
48
  xssmap -u https://target.tld --crawl --depth 3 --max-pages 200
@@ -148,7 +151,11 @@ def build_parser() -> argparse.ArgumentParser:
148
151
  help="never submit POST forms (overrides --autonomous)")
149
152
  scan.add_argument("--headers-points", action="store_true",
150
153
  help="test User-Agent/Referer/X-Forwarded-Host reflection")
151
- scan.add_argument("--threads", type=int, default=5, help="worker threads (default 5)")
154
+ scan.add_argument("--threads", type=int, default=10,
155
+ help="payload-testing workers (default 10)")
156
+ scan.add_argument("--crawl-threads", type=int, default=0, metavar="N",
157
+ help="discovery workers for the crawl (default: 16, or --threads "
158
+ "if larger)")
152
159
  scan.add_argument("--delay", type=float, default=0.0, help="minimum delay between requests")
153
160
  scan.add_argument("--timeout", type=float, default=15.0, help="per-request timeout")
154
161
  scan.add_argument("--retries", type=int, default=2, help="retries on transport errors")
@@ -390,6 +397,12 @@ def finalize_args(args: argparse.Namespace) -> None:
390
397
  args.crawl = False
391
398
  elif args.crawl is None:
392
399
  args.crawl = True
400
+ if getattr(args, "crawl_threads", 0) <= 0:
401
+ # Discovery only fetches the site's own links, and the shared throttle
402
+ # already backs off on 429/503, so it does not need to run at the same
403
+ # narrow width as payload testing. Measured on a 4096-page lab target:
404
+ # 5 workers 46 s, 16 workers 15 s, 32 workers 7 s.
405
+ args.crawl_threads = max(DEFAULT_CRAWL_THREADS, args.threads)
393
406
 
394
407
 
395
408
  def read_wordlist(path: str) -> list[str]:
@@ -794,11 +807,11 @@ def main(argv=None) -> int:
794
807
  if do_crawl:
795
808
  hosts_note = f", {len(seeds)} host(s)" if len(seeds) > 1 else ""
796
809
  log.action(f"crawling (depth {args.crawl_depth}, max {args.max_pages} pages, "
797
- f"{args.threads} threads{hosts_note})")
810
+ f"{args.crawl_threads} workers{hosts_note})")
798
811
  crawler = Crawler(
799
812
  http, scope, log,
800
813
  max_depth=args.crawl_depth, max_pages=args.max_pages,
801
- concurrency=args.threads, exclude=args.exclude, include=args.include,
814
+ concurrency=args.crawl_threads, exclude=args.exclude, include=args.include,
802
815
  crawl_js=not args.no_js, crawl_forms=True, ajax=not args.no_js,
803
816
  respect_robots=args.robots, headers=headers,
804
817
  )
@@ -489,8 +489,12 @@ class Crawler:
489
489
  self.result.param_urls.append(url)
490
490
 
491
491
  def _finish_param_urls(self):
492
- # sort so the shallow/short URLs get scanned first
493
- self.result.param_urls.sort(key=lambda u: (len(urllib.parse.urlsplit(u).query), len(u)))
492
+ # Shallow/short URLs get scanned first. The URL itself is the final key:
493
+ # without it, ties were broken by crawl completion order, so the same
494
+ # target produced a different point order (and a different --max-points
495
+ # cut) on every run.
496
+ self.result.param_urls.sort(
497
+ key=lambda u: (len(urllib.parse.urlsplit(u).query), len(u), u))
494
498
 
495
499
 
496
500
  def _title_of(text: str) -> str:
@@ -394,6 +394,11 @@ class Scanner:
394
394
  self.skip_points: set[str] = set(skip_points or ())
395
395
  self.completed_points: set[str] = set()
396
396
  self.resumed_points = 0
397
+ self._baseline_status_cache: dict[str, int] = {}
398
+ # One in-flight baseline per request: two parameters of the same URL are
399
+ # queued next to each other, so without this the workers both miss the
400
+ # cache and both fetch the same page.
401
+ self._baseline_inflight: dict[str, threading.Event] = {}
397
402
  self.profile_enabled = profile_enabled
398
403
  self.mutation_budget = max(0, mutation_budget)
399
404
  self.interrupted = False
@@ -544,6 +549,45 @@ class Scanner:
544
549
  self.completed_points.add(key)
545
550
  return found
546
551
 
552
+ def _baseline_status(self, req: Request) -> int | None:
553
+ """Status of a baseline request, cached per method+URL+body.
554
+
555
+ A URL with three parameters produces three points whose baseline request
556
+ is byte-identical, and the engine only reads ``.status`` from it. Bodies
557
+ are never cached, so this costs a dictionary of integers rather than a
558
+ copy of every page.
559
+ """
560
+ key = f"{req.method} {req.url}\x00{req.body!r}"
561
+ event: threading.Event
562
+ with self.lock:
563
+ cached = self._baseline_status_cache.get(key)
564
+ if cached is not None:
565
+ return cached
566
+ existing = self._baseline_inflight.get(key)
567
+ owner = existing is None
568
+ if existing is None:
569
+ event = threading.Event()
570
+ self._baseline_inflight[key] = event
571
+ else:
572
+ event = existing
573
+ if not owner:
574
+ # Another worker is fetching exactly this baseline; its answer is ours.
575
+ event.wait(getattr(self.http, "timeout", 15.0) + 5.0)
576
+ with self.lock:
577
+ return self._baseline_status_cache.get(key)
578
+ try:
579
+ status: int | None = self._send(req).status
580
+ except NetError:
581
+ status = None
582
+ with self.lock:
583
+ if status is not None:
584
+ if len(self._baseline_status_cache) >= 4096:
585
+ self._baseline_status_cache.clear()
586
+ self._baseline_status_cache[key] = status
587
+ self._baseline_inflight.pop(key, None)
588
+ event.set()
589
+ return status
590
+
547
591
  def _point_key(self, p: InjectionPoint) -> str:
548
592
  if isinstance(p, QueryPoint):
549
593
  return f"q:{p.url.split('#')[0]}:{p.index}"
@@ -573,21 +617,24 @@ class Scanner:
573
617
  else:
574
618
  value_original = original
575
619
 
576
- # 1. baseline with the original value (some apps 500 on empty params)
577
- try:
578
- baseline = self._send(point.build(value_original))
579
- except NetError:
580
- baseline = None
620
+ # 1. baseline with the original value (some apps 500 on empty params).
621
+ # Only the status is ever consulted, so it is shared between points that
622
+ # build the identical request -- the second parameter of a URL does not
623
+ # need its own copy of the same round trip.
624
+ baseline_status = self._baseline_status(point.build(value_original))
581
625
 
582
626
  marker = new_marker()
583
627
  marker_value = f"{marker}zqx"
584
628
  resp = self._send(point.build(marker_value))
585
- if baseline is not None and resp.status >= 500 and baseline.status < 500:
586
- # parameter breaks the app; try empty string as neutral baseline
587
- try:
588
- resp = self._send(point.build(marker_value))
589
- except NetError:
590
- return []
629
+ # A 500 here where the unmodified URL answered fine means our value broke
630
+ # the request. The old code answered that by re-sending the identical
631
+ # probe, which can only reproduce the 500 and costs a round trip; the
632
+ # error body is analysed below instead, because some applications echo
633
+ # the value inside their error page.
634
+ if (self.log and baseline_status is not None and resp.status >= 500
635
+ and baseline_status < 500):
636
+ self.log.debug(f" {point.label()} answers {resp.status} with our value "
637
+ f"({baseline_status} unmodified); error body kept")
591
638
 
592
639
  marker, resp, match = self._settle_probe(point, marker, resp)
593
640
  text = resp.text
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xssmap
3
- Version: 2.9.0
3
+ Version: 2.9.1
4
4
  Summary: Context-aware XSS scanner: crawler, browser verification, mXSS, charset and stored-XSS stages
5
5
  Author: Ian Mattas
6
6
  License-Expression: MIT
@@ -511,6 +511,23 @@ request. Three things make the result usable rather than noisy:
511
511
  A flagged version is a lead: it says the build has a published problem, not that this
512
512
  host is exploitable through it. The finding standard for XSS is unchanged.
513
513
 
514
+ ### Crawl and scan speed
515
+
516
+ Both phases are round-trip bound: the work per URL is fixed, so the wall-clock
517
+ cost is essentially `pages / workers x latency`. On a local lab of 4,096 linked
518
+ pages with 50 ms of latency per response, the same code, same page set and the same
519
+ 8,252 requests finished in **85 s with the old defaults (5 workers) and 38 s with
520
+ the current ones** (16 discovery workers, 10 payload workers). At a real WAN
521
+ latency of 200-300 ms the difference is the same multiple, in minutes.
522
+
523
+ Discovery therefore has its own worker count. It only fetches pages the site
524
+ links to itself, and the shared throttle still backs off on 429/503, so it does
525
+ not need to run as narrow as payload testing. Raise it with `--crawl-threads`;
526
+ `--threads` remains the knob for the scan phase. Two scan-side savings are
527
+ always on: the baseline response is fetched once per URL instead of once per
528
+ parameter, and the point order is deterministic, so `--max-points` cuts the same
529
+ places on every run.
530
+
514
531
  ### Browser cost
515
532
 
516
533
  `--browser` reuses one browser and one page per run. The report's `browser_verification`
@@ -571,7 +588,8 @@ per check, so the price of execution proof is visible rather than implied.
571
588
  | `--dom` / `--no-dom`, `--dom-flows N` | DOM analysis and its flow cap (default 15) |
572
589
  | `--headless new\|old` | headless mode passed to the browser |
573
590
  | `--screenshots` | capture a PNG for verified findings |
574
- | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability |
591
+ | `--threads N`, `--delay S`, `--timeout S`, `--retries N` | pacing and reliability (`--threads` sizes payload testing, default 10) |
592
+ | `--crawl-threads N` | discovery workers for the crawl; default 16, or `--threads` when that is larger |
575
593
  | `--proxy URL`, `--insecure` | `http://`/`socks5://` proxy for scanner and browser; skip TLS verification |
576
594
  | `--max-points N` | stop after N injection points |
577
595
  | `--state FILE` | resume file: skip points already completed, carry findings over |
@@ -767,7 +785,7 @@ response shape shows up as a miss on the other.
767
785
  ## Tests and CI
768
786
 
769
787
  ```bash
770
- python -m unittest discover -s tests -p "test_*.py" -v # 445 tests, no network
788
+ python -m unittest discover -s tests -p "test_*.py" -v # 490 tests, no network
771
789
  python -m ruff check . # lint
772
790
  python -m mypy xssmap # types on shipped package
773
791
  ```
@@ -867,7 +885,9 @@ browser harness on Linux (advisory, since headless timing on shared runners vari
867
885
  by at most the remaining slice (floored at 0.25s), and the report is then marked
868
886
  `partial` rather than pretending the sweep finished.
869
887
  - Heavy crawls against large sites should be bounded with `--max-pages`, `--delay` and
870
- `--threads`; the scanner honours those limits for every request it makes.
888
+ `--crawl-threads`; the scanner honours those limits for every request it makes.
889
+ A crawl of N pages is still N round trips: at 300 ms of latency and 16 workers that
890
+ is about a minute per 4,000 pages, and no amount of tuning changes that arithmetic.
871
891
  - WAF evasion is signature-driven at first and adaptive afterwards: each strategy that
872
892
  makes a payload reflect verbatim is tried first on later points, and one that only
873
893
  produces mangled or blocked responses moves to the end of the budget (the per-scan
@@ -23,6 +23,7 @@ tests/test_ops.py
23
23
  tests/test_params.py
24
24
  tests/test_recon_report.py
25
25
  tests/test_session.py
26
+ tests/test_speed.py
26
27
  tests/test_subdomains.py
27
28
  tests/test_tech.py
28
29
  tests/test_units.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes