netscraper 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. netscraper/__init__.py +9 -0
  2. netscraper/cli.py +635 -0
  3. netscraper/core/__init__.py +0 -0
  4. netscraper/core/banner.py +369 -0
  5. netscraper/core/cert_chain_parser.py +51 -0
  6. netscraper/core/cert_chain_rules.py +97 -0
  7. netscraper/core/cert_chain_runner.py +73 -0
  8. netscraper/core/concurrency.py +98 -0
  9. netscraper/core/cors_parser.py +45 -0
  10. netscraper/core/cors_rules.py +71 -0
  11. netscraper/core/cors_runner.py +100 -0
  12. netscraper/core/dirsearch_parser.py +48 -0
  13. netscraper/core/dirsearch_rules.py +39 -0
  14. netscraper/core/dirsearch_runner.py +109 -0
  15. netscraper/core/dns_parser.py +34 -0
  16. netscraper/core/dns_rules.py +57 -0
  17. netscraper/core/dns_runner.py +68 -0
  18. netscraper/core/excel_report.py +84 -0
  19. netscraper/core/families.py +467 -0
  20. netscraper/core/ftp_parser.py +28 -0
  21. netscraper/core/ftp_rules.py +56 -0
  22. netscraper/core/ftp_runner.py +63 -0
  23. netscraper/core/http_parser.py +59 -0
  24. netscraper/core/http_rules.py +81 -0
  25. netscraper/core/http_runner.py +118 -0
  26. netscraper/core/ldap_parser.py +53 -0
  27. netscraper/core/ldap_rules.py +109 -0
  28. netscraper/core/ldap_runner.py +100 -0
  29. netscraper/core/mssql_parser.py +26 -0
  30. netscraper/core/mssql_rules.py +35 -0
  31. netscraper/core/mssql_runner.py +70 -0
  32. netscraper/core/poc_renderer.py +399 -0
  33. netscraper/core/port_gate.py +79 -0
  34. netscraper/core/rpc_parser.py +59 -0
  35. netscraper/core/rpc_rules.py +86 -0
  36. netscraper/core/rpc_runner.py +82 -0
  37. netscraper/core/rules.py +234 -0
  38. netscraper/core/scan_worker.py +182 -0
  39. netscraper/core/smb_parser.py +69 -0
  40. netscraper/core/smb_rules.py +121 -0
  41. netscraper/core/smb_runner.py +123 -0
  42. netscraper/core/snmp_parser.py +23 -0
  43. netscraper/core/snmp_pure_probe.py +201 -0
  44. netscraper/core/snmp_rules.py +38 -0
  45. netscraper/core/snmp_runner.py +95 -0
  46. netscraper/core/ssh_parser.py +50 -0
  47. netscraper/core/ssh_rules.py +76 -0
  48. netscraper/core/ssh_runner.py +91 -0
  49. netscraper/core/sslscan_parser.py +140 -0
  50. netscraper/core/sslscan_runner.py +145 -0
  51. netscraper/core/targets.py +96 -0
  52. netscraper/core/terminal_capture.py +112 -0
  53. netscraper/core/theme.py +97 -0
  54. netscraper/wordlists/snmp_communities.txt +16 -0
  55. netscraper-0.1.0.dist-info/METADATA +695 -0
  56. netscraper-0.1.0.dist-info/RECORD +60 -0
  57. netscraper-0.1.0.dist-info/WHEEL +5 -0
  58. netscraper-0.1.0.dist-info/entry_points.txt +2 -0
  59. netscraper-0.1.0.dist-info/licenses/LICENSE +21 -0
  60. netscraper-0.1.0.dist-info/top_level.txt +1 -0
netscraper/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """
2
+ NetScraper -- automated network pentest check orchestration.
3
+
4
+ See cli.py for the entrypoint (installed as the `netscraper` console
5
+ script) and README.md for full usage.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ __version__ = "0.1.0"
netscraper/cli.py ADDED
@@ -0,0 +1,635 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ NetScraper -- CLI entrypoint.
4
+
5
+ Runs vuln-first, not host-first: every check's required port (SSH/22,
6
+ SMB/445, LDAP/389, ...) gets gated across the WHOLE target list up front,
7
+ in one unified port scan (a fast parallel TCP connect check -- see
8
+ core/port_gate.py), before anything else happens. That scan's real
9
+ open/closed counts then drive an interactive menu -- TLS, SSH, SMB, ...,
10
+ or All -- so you pick exactly the check(s) that actually apply to this
11
+ target list instead of always running everything. Only after that does
12
+ any real protocol-specific probe run, and only against hosts that actually
13
+ answered on the relevant port -- a host without SMB open never gets an SMB
14
+ check attempted; it's recorded as N/A, not scanned, not guessed at, and
15
+ not lumped in with a "safe" result it didn't earn.
16
+
17
+ Usage:
18
+ # Live scan (needs the relevant tools installed + real network reach).
19
+ # --output defaults to ./results if omitted. With no --modes flag, you'll
20
+ # be prompted to pick which check(s) to run once the port scan is done.
21
+ # After a scan finishes (or is interrupted with Ctrl-C), you're dropped
22
+ # back into the same Individual/Bulk/Exit main menu -- the tool only
23
+ # exits when you actually choose Exit there.
24
+ python3 cli.py --targets targets.txt
25
+
26
+ # Single target on the command line
27
+ python3 cli.py --target 192.168.1.10 --output ./results
28
+
29
+ # Skip the interactive picker and run specific check(s) non-interactively
30
+ python3 cli.py --targets targets.txt --modes tls,ssh
31
+
32
+ # Replay previously-captured output instead of hitting the network
33
+ # (used for testing / reprocessing evidence; see fixtures/)
34
+ python3 cli.py --targets targets_example.txt --output ./results --xml-dir ./fixtures
35
+
36
+ Output layout (per check):
37
+ <output>/SSL_Scan/Expired_Certificate/<host>.png (+ .txt)
38
+ <output>/SSL_Scan/Deprecated_TLS_Protocols/<host>.png (+ .txt)
39
+ <output>/SSL_Scan/Weak_Ciphers/<host>.png (+ .txt)
40
+ <output>/SSL_Scan/Weak_Key_Strength/<host>.png (+ .txt)
41
+ <output>/SSL_Scan/Hostname_Mismatch/<host>.png (+ .txt)
42
+ <output>/SSL_Scan/Untrusted_Self_Signed/<host>.png (+ .txt)
43
+ <output>/SSL_Scan/Incomplete_Certificate_Chain/<host>.png (+ .txt)
44
+ <output>/SSL_Scan/Secure_POC/<host>.png (+ .txt)
45
+ <output>/SSH/Weak_Deprecated_Crypto/<host>.png (+ .txt)
46
+ <output>/SSH/Banner_Disclosure/<host>.png (+ .txt)
47
+ <output>/SSH/Secure_POC/<host>.png (+ .txt)
48
+ <output>/raw/<check>/<host>_<artifact>.txt <- always saved, every check
49
+ <output>/summary.csv <- one row per (host, check)
50
+ <output>/PT_Scan_Report.xlsx <- styled Vulnerable/Not-
51
+ Vulnerable report, one
52
+ row per confirmed finding
53
+ or clean (host, check)
54
+
55
+ Every POC .png is the REAL captured tool output for that check (colors
56
+ preserved where the tool has them), with a red box around the specific
57
+ line(s) relevant to that finding. Nothing in the captured text is altered.
58
+ """
59
+ from __future__ import annotations
60
+
61
+ import argparse
62
+ import csv
63
+ import secrets
64
+ import sys
65
+ import time
66
+ from concurrent.futures import ProcessPoolExecutor, as_completed
67
+ from datetime import datetime, timezone
68
+ from pathlib import Path
69
+
70
+ from netscraper.core.targets import Target, expand_targets, load_targets_file
71
+ from netscraper.core.port_gate import gate_targets, gate_targets_own_port
72
+ from netscraper.core.families import FAMILIES
73
+ from netscraper.core.concurrency import worker_count
74
+ from netscraper.core.scan_worker import run_one_target_check
75
+ from netscraper.core.excel_report import write_report
76
+ from netscraper.core.banner import clear_screen, print_banner, prompt_mode, prompt_scan_modes, prompt_port_list, ReturnToMainMenu
77
+ from netscraper.core.theme import title, accent, info, ok, bad, warn, err, muted, prompt, host as c_host
78
+
79
+
80
+ def parse_args(argv=None) -> argparse.Namespace:
81
+ p = argparse.ArgumentParser(description="NetScraper")
82
+ src = p.add_mutually_exclusive_group(required=False)
83
+ src.add_argument("--targets", help="Path to a targets file (IPs, hostnames, or CIDR subnets, one per line)")
84
+ src.add_argument("--target", help="A single IP, hostname, or CIDR subnet")
85
+ p.add_argument("--output", default="./results",
86
+ help="Root output directory for POCs, raw evidence, and summary.csv (default: ./results). "
87
+ "Each run gets its own uniquely-named subfolder under this root (see --no-run-id) so "
88
+ "repeated scans never overwrite each other's evidence.")
89
+ p.add_argument("--no-run-id", action="store_true",
90
+ help="Write directly into --output instead of a per-run unique subfolder (will overwrite "
91
+ "a previous run's evidence at that same path -- use this only for scripted/CI use "
92
+ "where you manage the output path yourself).")
93
+ p.add_argument("--xml-dir", default=None,
94
+ help="Replay mode: read pre-captured evidence from this directory instead of scanning live")
95
+ p.add_argument("--timeout", type=int, default=90,
96
+ help="Per-host, per-tool timeout in seconds (live mode only). A real live TLS "
97
+ "scan tests dozens of cipher/protocol combos and can legitimately take longer "
98
+ "than a quick local test, so this is intentionally generous; sslscan's own "
99
+ "internal per-probe timeouts are separately capped so a single slow probe "
100
+ "can't blow past this on its own (default: 90)")
101
+ p.add_argument("--modes", "--families", dest="families", default=None, metavar="MODES",
102
+ help="Comma-separated check mode names to run (default: prompts interactively after the "
103
+ "port scan, or 'all' if run non-interactively without this flag). Available: "
104
+ + ",".join(f["name"] for f in FAMILIES))
105
+ p.add_argument("--port-gate-timeout", type=float, default=3.0,
106
+ help="TCP connect timeout in seconds for the port-gate pre-check (default: 3.0)")
107
+ p.add_argument("--mode-pause", "--family-pause", dest="family_pause", type=int, default=5, metavar="SECONDS",
108
+ help="Seconds to pause between checks during a LIVE scan, once one check "
109
+ "finishes and before the next one starts (default: 5; ignored in --xml-dir replay mode, "
110
+ "and skipped after the last check). Set to 0 to disable.")
111
+ p.add_argument("--dirsearch-ports", default=None, metavar="PORTS",
112
+ help="Comma-separated port(s) for the DIRSEARCH mode to brute-force, e.g. 80,443,8080 "
113
+ "(single port also accepted, e.g. 80). DIRSEARCH has no fixed port like every other "
114
+ "check, so this is required if --modes/--families includes 'dirsearch' for a scripted "
115
+ "run; in the interactive picker, you're simply asked for it right after choosing that "
116
+ "mode if this isn't given up front.")
117
+ p.add_argument("--dirsearch-timeout", type=int, default=300, metavar="SECONDS",
118
+ help="Overall wall-clock budget in seconds for one DIRSEARCH brute-force run against a "
119
+ "single host:port (default: 300). Kept separate from --timeout since a real "
120
+ "directory/file brute-force legitimately takes a lot longer than the other checks.")
121
+ p.add_argument("--cors-ports", default=None, metavar="PORTS",
122
+ help="Comma-separated port(s) for the CORS mode to probe, e.g. 80,443,8080 (single port "
123
+ "also accepted, e.g. 443). CORS has no fixed port like every other check, so this is "
124
+ "required if --modes/--families includes 'cors' for a scripted run; in the interactive "
125
+ "picker, you're simply asked for it right after choosing that mode if this isn't given "
126
+ "up front.")
127
+ return p.parse_args(argv)
128
+
129
+
130
+ def _parse_port_list(raw: str) -> list[int]:
131
+ """Turns "80" or "80,81,443,8080" into a de-duplicated, order-preserving
132
+ list of real port numbers (1-65535); silently skips anything that
133
+ isn't a valid port instead of erroring, so one typo doesn't blow up an
134
+ otherwise-good list -- the caller checks for an empty result instead."""
135
+ ports: list[int] = []
136
+ seen = set()
137
+ for token in raw.split(","):
138
+ token = token.strip()
139
+ if not token.isdigit():
140
+ continue
141
+ value = int(token)
142
+ if 1 <= value <= 65535 and value not in seen:
143
+ seen.add(value)
144
+ ports.append(value)
145
+ return ports
146
+
147
+
148
+ # Any check with no fixed port of its own (currently DIRSEARCH and CORS)
149
+ # needs its --xxx-ports CLI flag and the "check" verb used in its interactive
150
+ # prompt looked up by family name -- kept in one small table instead of
151
+ # scattering per-family if/elif chains through _resolve_prompt_gated_ports.
152
+ _PROMPT_GATED_FAMILIES = {
153
+ "dirsearch": {"cli_attr": "dirsearch_ports", "flag_name": "--dirsearch-ports", "verb": "brute-force"},
154
+ "cors": {"cli_attr": "cors_ports", "flag_name": "--cors-ports", "verb": "check"},
155
+ }
156
+
157
+
158
+ def _resolve_prompt_gated_ports(args: argparse.Namespace, targets: list, families: list, port_scan: dict) -> int:
159
+ """
160
+ Some checks (DIRSEARCH, CORS, ...) have no fixed port like every other
161
+ check -- the port(s) to use are only known once the user says so,
162
+ either via that family's --xxx-ports flag up front or, in the
163
+ interactive picker, right after choosing the mode. For every such
164
+ family that was actually selected, this fills in port_scan[name] with a
165
+ real quick TCP gate across every (host, chosen port) combination from
166
+ the target list -- the exact same kind of pre-check every other family
167
+ gets, just computed later because the ports weren't known until now.
168
+
169
+ Returns 0 on success (including "no prompt-gated family was selected,
170
+ nothing to do"), or 1 if a family had to be dropped because no valid
171
+ port(s) could be resolved for a scripted run -- matching run_one_scan's
172
+ own exit-code convention. Can raise ReturnToMainMenu (from
173
+ prompt_port_list, interactive-only) -- deliberately left to propagate
174
+ untouched, same as prompt_scan_modes.
175
+ """
176
+ for name, spec in _PROMPT_GATED_FAMILIES.items():
177
+ if not any(f["name"] == name for f in families):
178
+ continue
179
+
180
+ label = name.upper()
181
+ cli_ports = getattr(args, spec["cli_attr"])
182
+
183
+ if cli_ports:
184
+ ports = _parse_port_list(cli_ports)
185
+ if not ports:
186
+ print(err(f"[!] {spec['flag_name']}={cli_ports!r} has no valid port number(s)."), file=sys.stderr)
187
+ return 1
188
+ elif args.families:
189
+ # Scripted run (--modes/--families given) with this family
190
+ # included, but nothing to prompt for -- there's no interactive
191
+ # session here.
192
+ print(err(f"[!] --modes includes '{name}' but no {spec['flag_name']} was given."), file=sys.stderr)
193
+ return 1
194
+ else:
195
+ while True:
196
+ raw = prompt_port_list(label, spec["verb"]) # may raise ReturnToMainMenu -- let it through untouched
197
+ ports = _parse_port_list(raw)
198
+ if ports:
199
+ break
200
+ print(warn(f"[!] No valid port number(s) found in '{raw}' -- try again."))
201
+
202
+ unique_hosts = list(dict.fromkeys(t.host for t in targets))
203
+ check_targets = [Target(host=h, port=p) for h in unique_hosts for p in ports]
204
+
205
+ print("\n" + info("[+] ") + accent(f"Quick port scan for {label} ") +
206
+ c_host(f"({', '.join(str(p) for p in ports)}) across {len(unique_hosts)} host(s) ..."))
207
+ if args.xml_dir:
208
+ qualifying = {t: True for t in check_targets}
209
+ else:
210
+ qualifying = gate_targets_own_port(check_targets, timeout=args.port_gate_timeout)
211
+ open_count = sum(1 for v in qualifying.values() if v)
212
+ print(muted(f" {label:<6} : ") + c_host(f"{open_count}/{len(check_targets)} host:port combo(s) open"))
213
+
214
+ port_scan[name] = {"targets": check_targets, "qualifying": qualifying, "is_udp": False}
215
+
216
+ return 0
217
+
218
+
219
+ def run_one_scan(args: argparse.Namespace) -> int:
220
+ """
221
+ Runs one target list's worth of scanning (target load -> unified port
222
+ scan -> mode selection -> per-check execution -> summary.csv +
223
+ PT_Scan_Report.xlsx) and returns an exit code.
224
+
225
+ Every check's per-host work runs across several hosts AT ONCE, in a
226
+ process pool sized automatically from this machine's own RAM/CPU (see
227
+ core/concurrency.py) -- fully automatic, with no flag or prompt
228
+ exposed for it anywhere. This is what keeps a 60-200 host target list
229
+ from taking as long as scanning every host one at a time; see
230
+ execute_families() below and core/scan_worker.py for how each host's
231
+ work is packaged up for a worker process and how results stream back
232
+ as each one finishes.
233
+
234
+ In interactive use (no --modes/--families given), the mode-selection
235
+ step is a loop, not a one-shot: after a chosen check (or several)
236
+ finishes running against this target list, the SAME check menu comes
237
+ back up so you can run another one against the SAME hosts without
238
+ re-entering the target/subnet from scratch -- summary.csv and
239
+ PT_Scan_Report.xlsx accumulate across every round in this same output
240
+ folder. "Back to main menu" (an explicit numbered option, or Ctrl-C,
241
+ same as ever) is how you actually leave to scan something else. A
242
+ scripted run (--modes/--families given) is unchanged: it runs that
243
+ fixed set once and returns.
244
+
245
+ Two things are caught here so a scan never just dies mid-run:
246
+ - A Ctrl-C that lands *during* the actual scanning (anywhere from the
247
+ port scan onward) never crashes with a traceback: whatever results
248
+ were already collected are still written out (partial progress is
249
+ never silently thrown away), then ReturnToMainMenu is raised so the
250
+ caller (main()) can decide what "going back" means -- looping to the
251
+ interactive menu, or just exiting cleanly for a scripted run.
252
+ - Any OTHER unexpected exception during scanning (a POC-render hiccup,
253
+ a resource issue under a big target list, anything not already
254
+ turned into a clean per-host ERROR row) is likewise caught, partial
255
+ results are saved the same way, and control returns to the menu
256
+ instead of the whole process crashing and losing everything that had
257
+ already completed. A per-target failure in POC rendering or evidence
258
+ writing is caught even more locally, inside execute_families() below,
259
+ so ONE bad screenshot only costs that one host/check an ERROR row,
260
+ not the rest of the scan.
261
+ A ReturnToMainMenu raised by prompt_scan_modes()/prompt_port_list()
262
+ themselves (i.e. deliberate navigation, not a real error) is
263
+ deliberately let straight through untouched, never treated as a crash.
264
+ """
265
+ if args.targets:
266
+ targets = load_targets_file(args.targets)
267
+ else:
268
+ targets = expand_targets([args.target])
269
+
270
+ if not targets:
271
+ print(err("[!] No valid targets found."), file=sys.stderr)
272
+ return 1
273
+
274
+ # If --modes/--families was given, only scan ports for that subset (no
275
+ # point port-scanning checks that were never going to run); otherwise
276
+ # scan ports for every check up front so the mode-selection menu below
277
+ # can show real, live open/closed counts for every option.
278
+ if args.families:
279
+ wanted = set(n.strip().lower() for n in args.families.split(","))
280
+ scan_universe = [f for f in FAMILIES if f["name"] in wanted]
281
+ if not scan_universe:
282
+ print(err(f"[!] No matching check modes for --modes={args.families}"), file=sys.stderr)
283
+ return 1
284
+ else:
285
+ scan_universe = FAMILIES
286
+
287
+ # Every run gets its own uniquely-named evidence folder (timestamp + a
288
+ # short random suffix so two runs started in the same second never
289
+ # collide) instead of always writing into the same fixed "results/" --
290
+ # that way a tester can run multiple scans (different targets, or a
291
+ # re-test after remediation) without one run silently overwriting the
292
+ # previous one's POCs/raw evidence. --no-run-id opts back out of this
293
+ # for scripted/CI use where the caller manages the output path itself.
294
+ run_id = datetime.now().strftime("%Y%m%d_%H%M%S") + "_" + secrets.token_hex(3)
295
+ output_root = Path(args.output) if args.no_run_id else Path(args.output) / f"scan_{run_id}"
296
+ summary_rows = [] # host, port, check, status, findings -- for summary.csv
297
+ excel_rows = [] # (vulnerability, host_str, "Vulnerable"/"Not Vulnerable") -- for PT_Scan_Report.xlsx
298
+
299
+ print(title("=" * 60))
300
+ print(info("[+] ") + accent("Target list loaded: ") + c_host(f"{len(targets)} IP/host(es) to scan"))
301
+ print(info("[+] ") + accent("Scan type: ") + (warn("REPLAY from " + args.xml_dir) if args.xml_dir else ok("LIVE")))
302
+ print(info("[+] ") + accent("Results folder: ") + c_host(str(output_root)))
303
+ print(title("=" * 60))
304
+
305
+ def finalize(interrupted: bool) -> int:
306
+ output_root.mkdir(parents=True, exist_ok=True)
307
+ summary_path = output_root / "summary.csv"
308
+ with open(summary_path, "w", newline="", encoding="utf-8") as fh:
309
+ writer = csv.writer(fh)
310
+ writer.writerow(["host", "port", "check", "status", "findings"])
311
+ writer.writerows(summary_rows)
312
+
313
+ report_path = output_root / "PT_Scan_Report.xlsx"
314
+ write_report(excel_rows, report_path)
315
+
316
+ if interrupted:
317
+ print(warn("[!] Partial results saved -- ") + accent("Summary: ") + c_host(str(summary_path)) +
318
+ warn(" | ") + accent("Report: ") + c_host(str(report_path)))
319
+ else:
320
+ print("\n" + ok("[+] Done.") + " Summary: " + accent(str(summary_path)) +
321
+ " | Report: " + accent(str(report_path)))
322
+ return 0
323
+
324
+ try:
325
+ # --- Unified port scan ---------------------------------------- #
326
+ # Every check that needs a TCP port (SSH/22, SMB/445, LDAP/389,
327
+ # RPC/445, MSSQL/1433, FTP/21, HTTP/80, TLS's own per-target port)
328
+ # gets gated ONCE here, up front, across the whole target list --
329
+ # instead of re-discovering "is this port even open" check-by-check
330
+ # with a pause in between. This also means the mode-selection menu
331
+ # below can show real open-port counts, so you can pick exactly the
332
+ # check(s) that actually apply to this target list instead of
333
+ # guessing.
334
+ print("\n" + info("[+] ") + accent("Scanning required ports across ") +
335
+ c_host(f"{len(targets)} target(s)") + accent(" ..."))
336
+ port_scan: dict = {}
337
+ for family in scan_universe:
338
+ name = family["name"]
339
+ gate_port = family["gate_port"]
340
+
341
+ # DIRSEARCH has no fixed port at all -- the user only supplies
342
+ # one (or several) after actually choosing this mode, so there's
343
+ # nothing to gate here yet. _resolve_prompt_gated_ports() fills
344
+ # port_scan["dirsearch"] in for real once that's known.
345
+ if gate_port == "prompt":
346
+ port_scan[name] = {"targets": [], "qualifying": {}, "is_udp": False, "deferred": True}
347
+ print(muted(f" {name.upper():<6} : port(s) chosen after you select this mode"))
348
+ continue
349
+
350
+ # A check whose gate_port is "udp:<port>" (e.g. DNS, SNMP) has
351
+ # no TCP pre-check at all -- there's nothing meaningful to
352
+ # "nc -z" for a UDP-only service, so every host is handed
353
+ # straight to the real protocol probe during the actual scan,
354
+ # and that probe's own success/timeout IS the gate.
355
+ is_udp = isinstance(gate_port, str) and gate_port.startswith("udp:")
356
+ fixed_port = int(gate_port.split(":", 1)[1]) if is_udp else gate_port
357
+
358
+ if is_udp or fixed_port is not None:
359
+ # Fixed-port check (e.g. SSH always on 22): the port a
360
+ # target string specified was almost certainly meant for a
361
+ # different check (TLS's own-port override), so probe
362
+ # host:<gate_port> instead, de-duplicated by host first --
363
+ # otherwise the same "192.168.1.10" entered once for TLS
364
+ # and once for a custom port would get re-probed for SSH
365
+ # twice for no reason.
366
+ unique_hosts = list(dict.fromkeys(t.host for t in targets))
367
+ check_targets = [Target(host=h, port=fixed_port) for h in unique_hosts]
368
+ else:
369
+ check_targets = targets
370
+
371
+ if is_udp:
372
+ qualifying = {t: True for t in check_targets}
373
+ elif args.xml_dir:
374
+ # In replay mode there's no real network to gate against --
375
+ # every target is treated as qualifying, and the replay
376
+ # fixture itself is the source of truth for what's
377
+ # "available".
378
+ qualifying = {t: True for t in check_targets}
379
+ else:
380
+ qualifying = (
381
+ gate_targets(check_targets, fixed_port, timeout=args.port_gate_timeout)
382
+ if fixed_port is not None
383
+ else gate_targets_own_port(check_targets, timeout=args.port_gate_timeout)
384
+ )
385
+
386
+ port_scan[name] = {"targets": check_targets, "qualifying": qualifying, "is_udp": is_udp}
387
+
388
+ label = name.upper()
389
+ if is_udp:
390
+ print(muted(f" {label:<6} (UDP) : checked directly during the scan (no TCP pre-check possible)"))
391
+ else:
392
+ open_count = sum(1 for v in qualifying.values() if v)
393
+ print(muted(f" {label:<6} : ") + c_host(f"{open_count}/{len(check_targets)} host(s) open"))
394
+
395
+ def build_mode_options() -> list[tuple[str, str]]:
396
+ options = []
397
+ for family in FAMILIES:
398
+ info_row = port_scan[family["name"]]
399
+ if info_row.get("deferred"):
400
+ desc = f"{family['name'].upper():<6} -- {family.get('deferred_desc', 'pick port(s) after selecting')}"
401
+ elif info_row["is_udp"]:
402
+ desc = f"{family['name'].upper():<6} -- UDP service, checked directly"
403
+ else:
404
+ open_count = sum(1 for v in info_row["qualifying"].values() if v)
405
+ desc = (f"{family['name'].upper():<6} -- {open_count}/{len(info_row['targets'])} host(s) "
406
+ f"with the required port open")
407
+ options.append((family["name"], desc))
408
+ return options
409
+
410
+ def execute_families(families: list, pool: ProcessPoolExecutor) -> None:
411
+ """
412
+ Runs the given families' checks against the already-gated
413
+ target list, appending to summary_rows/excel_rows and writing
414
+ every POC/evidence file as it goes.
415
+
416
+ Every qualifying (host, check) pair for the CURRENT family is
417
+ handed to the shared worker process pool at once -- this is
418
+ the actual "split the IP list into batches and scan several at
419
+ the same time" behavior: several hosts' real checks run
420
+ simultaneously, in independent worker processes, instead of
421
+ one at a time. Results are collected as each host finishes
422
+ (netscraper.core.scan_worker.run_one_target_check does the real work and
423
+ returns a plain-data dict -- nothing shared, so results from
424
+ many workers merge safely here regardless of completion
425
+ order), and each host's line prints the moment it's done --
426
+ so progress is visibly streaming in throughout, never a long
427
+ silent gap followed by everything appearing at once.
428
+
429
+ A per-target failure -- the check's own run() reporting
430
+ status="error", an unexpected exception from POC rendering or
431
+ evidence writing, or even a worker process dying outright --
432
+ is turned into this one host/check's own ERROR row inside
433
+ run_one_target_check (or, for an outright worker crash, right
434
+ here) instead of ever taking down the rest of the batch.
435
+ """
436
+ for family_index, family in enumerate(families):
437
+ name = family["name"]
438
+ folder_name = family["folder"]
439
+ precomputed = port_scan[name]
440
+ family_targets = precomputed["targets"]
441
+ qualifying = precomputed["qualifying"]
442
+ is_udp_family = precomputed["is_udp"]
443
+ render_this = family.get("render_poc", True)
444
+ # DIRSEARCH's real brute-force run legitimately takes a lot
445
+ # longer than the other checks' quick probes -- give it its
446
+ # own, more generous wall-clock budget instead of the shared
447
+ # --timeout.
448
+ run_timeout = args.dirsearch_timeout if name == "dirsearch" else args.timeout
449
+
450
+ print("\n" + info("[+] ") + "Mode: " + accent(name.upper()))
451
+ if is_udp_family:
452
+ print(muted(f" UDP-based service -- no TCP pre-check; the real protocol probe determines "
453
+ f"reachability for each of {len(family_targets)} host(s)"))
454
+ else:
455
+ open_count = sum(1 for v in qualifying.values() if v)
456
+ print(info(f" {open_count}/{len(family_targets)} host(s) have the required port open"))
457
+
458
+ jobs = []
459
+ for target in family_targets:
460
+ if not qualifying.get(target):
461
+ summary_rows.append([target.host, target.port, name, "N/A", "port not open"])
462
+ continue
463
+ jobs.append({
464
+ "family_name": name, "host": target.host, "port": target.port,
465
+ "xml_dir": args.xml_dir, "run_timeout": run_timeout,
466
+ "output_root": str(output_root), "folder": folder_name,
467
+ "render_this": render_this,
468
+ })
469
+
470
+ if jobs:
471
+ print(muted(f" [*] {len(jobs)} host(s) queued -- scanning in parallel ..."))
472
+ futures = {pool.submit(run_one_target_check, job): job for job in jobs}
473
+ done_count = 0
474
+ for future in as_completed(futures):
475
+ done_count += 1
476
+ job = futures[future]
477
+ try:
478
+ result = future.result()
479
+ except Exception as exc: # noqa: BLE001
480
+ # The worker process itself died (as opposed to
481
+ # the check inside it failing cleanly, which
482
+ # run_one_target_check already turns into a
483
+ # normal ERROR result) -- still record it as
484
+ # this one host/check's own ERROR row instead
485
+ # of losing the rest of the batch over it.
486
+ host_str = f"{job['host']}:{job['port']}"
487
+ print(muted(" [*] ") + c_host(host_str) + muted(" ... ") +
488
+ err(f"ERROR (worker crashed: {exc})") + muted(f" ({done_count}/{len(jobs)})"))
489
+ summary_rows.append([job["host"], job["port"], name, "ERROR", f"worker crashed: {exc}"])
490
+ continue
491
+ summary_rows.extend(result["summary_rows"])
492
+ excel_rows.extend(result["excel_rows"])
493
+ print(result["line"] + muted(f" ({done_count}/{len(jobs)})"))
494
+
495
+ if args.family_pause > 0 and not args.xml_dir and family_index < len(families) - 1:
496
+ print("\n" + info("[+] ") + f"{accent(name.upper())} mode complete. " +
497
+ muted(f"Pausing {args.family_pause}s before the next mode ..."))
498
+ time.sleep(args.family_pause)
499
+
500
+ # Host-level parallelism: several hosts' checks run at once, in
501
+ # independent worker processes, instead of one host at a time --
502
+ # this is what actually makes a 60-200 host target list come back
503
+ # fast instead of taking as long as running every host in series.
504
+ # Sized automatically from this machine's own RAM/CPU (see
505
+ # core/concurrency.py) -- there is deliberately no flag or prompt
506
+ # for this anywhere; it just happens the moment a target list is
507
+ # given. Must be a PROCESS pool, not a thread pool: sslscan/
508
+ # ssh-audit's pty capture (core/terminal_capture.py) uses
509
+ # os.fork() directly, which is unsafe from multiple threads in one
510
+ # process, and Playwright's sync API is single-thread-only -- a
511
+ # process pool sidesteps both since every worker is a fully
512
+ # independent OS process. One pool is created ONCE for the whole
513
+ # run (reused across every check and every round of the
514
+ # interactive menu below) so each worker's own lazily-launched
515
+ # Chromium (see netscraper.core.poc_renderer.get_process_browser(), used
516
+ # inside core/scan_worker.py) gets reused for every POC that
517
+ # worker ever renders, not relaunched per check.
518
+ pool_size = worker_count(len(targets))
519
+ print("\n" + info("[+] ") + accent("Parallel scan workers: ") +
520
+ c_host(f"{pool_size} (auto-sized for this machine)"))
521
+ pool = ProcessPoolExecutor(max_workers=pool_size)
522
+ try:
523
+ if args.families:
524
+ # --- Choose which check(s) to actually run (scripted) --- #
525
+ families = scan_universe
526
+ print("\n" + info("[+] ") + accent("Modes queued (from --modes): ") +
527
+ c_host(f"{len(families)} ({', '.join(f['name'].upper() for f in families)})"))
528
+
529
+ rc = _resolve_prompt_gated_ports(args, targets, families, port_scan)
530
+ if rc != 0:
531
+ return rc
532
+
533
+ execute_families(families, pool)
534
+ else:
535
+ # --- Interactive picker, looping on this same target list --- #
536
+ # After finishing one round of checks, offer the same menu
537
+ # again instead of forcing a trip all the way back through
538
+ # Individual/Bulk/file re-entry just to run one more check
539
+ # against the SAME hosts -- e.g. TLS first, then SSH, then
540
+ # SMB, all without re-entering the target list each time.
541
+ # "Back to main menu" (or Ctrl-C, same as ever) is how you
542
+ # actually leave to scan something else.
543
+ while True:
544
+ print()
545
+ options = build_mode_options()
546
+ chosen_names = prompt_scan_modes(options, allow_back=True) # may raise ReturnToMainMenu
547
+ families = [f for f in FAMILIES if f["name"] in chosen_names]
548
+ print(info("[+] ") + accent("Modes selected: ") +
549
+ c_host(f"{len(families)} ({', '.join(f['name'].upper() for f in families)})"))
550
+
551
+ rc = _resolve_prompt_gated_ports(args, targets, families, port_scan)
552
+ if rc != 0:
553
+ print(err("[!] Nothing to run for that selection -- pick again."))
554
+ continue
555
+
556
+ execute_families(families, pool)
557
+ finalize(interrupted=False) # keep summary.csv/xlsx current after every round
558
+ print("\n" + info("[+] ") +
559
+ accent(f"{len(families)} mode(s) complete against this target list.") +
560
+ muted(" Pick another check, or 'back' for the main menu."))
561
+ finally:
562
+ # cancel_futures=True drops anything still queued but not yet
563
+ # started (relevant on a Ctrl-C/error path -- no point starting
564
+ # new worker tasks once we're on our way out); wait=True still
565
+ # lets whatever's already RUNNING in a worker right now finish
566
+ # cleanly rather than killing those processes mid-check, so a
567
+ # scan never leaves orphaned worker processes behind.
568
+ pool.shutdown(wait=True, cancel_futures=True)
569
+
570
+ except KeyboardInterrupt:
571
+ print("\n" + warn("[!] Scan interrupted by user (Ctrl-C)."))
572
+ finalize(interrupted=True)
573
+ raise ReturnToMainMenu()
574
+ except ReturnToMainMenu:
575
+ raise
576
+ except Exception as exc: # noqa: BLE001
577
+ # Something we didn't anticipate blew up mid-scan (e.g. a resource
578
+ # exhaustion issue, an unhandled tool edge case) -- never lose
579
+ # already-collected results over it the way a bare traceback would.
580
+ # Save what we have, say plainly what happened, and bounce back to
581
+ # the menu (or exit cleanly for a scripted run) instead of a hard
582
+ # crash taking the whole process down mid-scan.
583
+ print("\n" + err(f"[!] Unexpected error during the scan: {exc}"))
584
+ finalize(interrupted=True)
585
+ raise ReturnToMainMenu()
586
+
587
+ return finalize(interrupted=False)
588
+
589
+
590
+ def main(argv=None) -> int:
591
+ args = parse_args(argv)
592
+ interactive = not args.targets and not args.target
593
+
594
+ if not interactive:
595
+ # Scripted/automated use (--targets or --target given up front):
596
+ # run exactly once and exit, same as before -- no menu loop, since
597
+ # there's no interactive session to loop back into. A Ctrl-C mid-
598
+ # scan still saves partial results (see run_one_scan) instead of
599
+ # crashing; there's just nothing to return to afterward, so it's
600
+ # reported as an interrupted run rather than looped.
601
+ clear_screen()
602
+ print_banner()
603
+ try:
604
+ return run_one_scan(args)
605
+ except ReturnToMainMenu:
606
+ return 130
607
+
608
+ # Interactive use (bare `python3 cli.py`, or with only --modes/--output/
609
+ # etc. given but no target source): loop on the main menu until the
610
+ # user explicitly chooses Exit (or Ctrl-C's at that menu itself) --
611
+ # every other Ctrl-C, and every completed scan, bounces back here
612
+ # instead of quitting the process.
613
+ while True:
614
+ clear_screen()
615
+ print_banner()
616
+ mode, value = prompt_mode() # raises SystemExit on Exit / Ctrl-C here
617
+
618
+ run_args = argparse.Namespace(**vars(args))
619
+ if mode == "targets":
620
+ run_args.targets = value
621
+ run_args.target = None
622
+ else:
623
+ run_args.target = value
624
+ run_args.targets = None
625
+
626
+ try:
627
+ run_one_scan(run_args)
628
+ except ReturnToMainMenu:
629
+ pass # a message was already printed; fall through to the menu again
630
+
631
+ print("\n" + info("[+] ") + accent("Returning to the main menu ..."))
632
+
633
+
634
+ if __name__ == "__main__":
635
+ sys.exit(main())
File without changes