netscraper 0.1.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {netscraper-0.1.0/src/netscraper.egg-info → netscraper-0.2.2}/PKG-INFO +39 -26
- {netscraper-0.1.0 → netscraper-0.2.2}/README.md +38 -25
- {netscraper-0.1.0 → netscraper-0.2.2}/pyproject.toml +1 -1
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/__init__.py +1 -1
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/cli.py +39 -13
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/poc_renderer.py +71 -1
- {netscraper-0.1.0 → netscraper-0.2.2/src/netscraper.egg-info}/PKG-INFO +39 -26
- {netscraper-0.1.0 → netscraper-0.2.2}/LICENSE +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/MANIFEST.in +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/setup.cfg +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/__init__.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/banner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cert_chain_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cert_chain_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cert_chain_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/concurrency.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cors_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cors_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/cors_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dirsearch_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dirsearch_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dirsearch_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dns_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dns_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/dns_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/excel_report.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/families.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ftp_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ftp_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ftp_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/http_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/http_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/http_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ldap_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ldap_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ldap_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/mssql_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/mssql_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/mssql_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/port_gate.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/rpc_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/rpc_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/rpc_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/scan_worker.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/smb_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/smb_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/smb_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/snmp_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/snmp_pure_probe.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/snmp_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/snmp_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ssh_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ssh_rules.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/ssh_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/sslscan_parser.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/sslscan_runner.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/targets.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/terminal_capture.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/core/theme.py +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper/wordlists/snmp_communities.txt +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper.egg-info/SOURCES.txt +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper.egg-info/dependency_links.txt +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper.egg-info/entry_points.txt +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper.egg-info/requires.txt +0 -0
- {netscraper-0.1.0 → netscraper-0.2.2}/src/netscraper.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: netscraper
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Automated network pentest check orchestration -- port-gates a target list, runs the relevant checks (TLS, SSH, SMB, LDAP, RPC, DNS, MSSQL, SNMP, FTP, HTTP, DIRSEARCH, CORS), and produces per-finding POC screenshots plus a styled Excel report.
|
|
5
5
|
Author: Wati Cyber
|
|
6
6
|
License: MIT
|
|
@@ -37,15 +37,18 @@ ready to drop straight into a report appendix.
|
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
39
|
pip install netscraper
|
|
40
|
-
playwright install chromium # one-time, for POC screenshots
|
|
41
40
|
netscraper --targets targets.txt
|
|
42
41
|
```
|
|
43
42
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
That's it — no separate `playwright install chromium` step: the first time
|
|
44
|
+
the tool actually needs Chromium for a POC screenshot, it notices it isn't
|
|
45
|
+
downloaded yet and fetches it automatically (a one-time, ~150MB download;
|
|
46
|
+
every run after that is an instant no-op check). For authorized
|
|
47
|
+
penetration testing and security research only — see the legal banner the
|
|
48
|
+
tool itself prints on every run. See "Requirements" below for the external
|
|
49
|
+
command-line tools (`sslscan`, `nmap`, etc.) each check shells out to;
|
|
50
|
+
`pip install` alone gets you the `netscraper` command and its Python
|
|
51
|
+
dependencies, not those OS-level tools.
|
|
49
52
|
|
|
50
53
|
Runs **vuln-first, not host-first**: every check's required port (TLS's own
|
|
51
54
|
port, SSH/22, SMB/445, LDAP/389, RPC/445, MSSQL/1433, FTP/21, HTTP/80) gets
|
|
@@ -358,14 +361,17 @@ signing flagged for manual verification, both real, independent signals).
|
|
|
358
361
|
|
|
359
362
|
- Python 3.9+, on macOS or Linux (uses a Unix pty for ANSI-preserving
|
|
360
363
|
terminal capture where relevant; no Windows support).
|
|
361
|
-
- `pip install netscraper
|
|
362
|
-
`
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
`
|
|
368
|
-
the
|
|
364
|
+
- `pip install netscraper` (once) — pulls in `ssh-audit`, `impacket` (for
|
|
365
|
+
`rpcdump.py`), `dirsearch`, `playwright` (renders POC screenshots),
|
|
366
|
+
`ansi2html` (real captured terminal colors → matching HTML), and
|
|
367
|
+
`openpyxl` (writes `PT_Scan_Report.xlsx`) as dependencies automatically,
|
|
368
|
+
and puts the `netscraper` command on your PATH. No separate `playwright
|
|
369
|
+
install chromium` step needed — see `ensure_chromium_installed()` in
|
|
370
|
+
`core/poc_renderer.py`, which the `netscraper` command calls on startup
|
|
371
|
+
and which downloads Chromium the first time it notices it isn't there
|
|
372
|
+
yet. Running from a source checkout instead of the published package?
|
|
373
|
+
`pip install -e .` from the repo root does the same thing against your
|
|
374
|
+
local copy.
|
|
369
375
|
- Command-line tools this toolkit shells out to (install via your package
|
|
370
376
|
manager — Debian/Ubuntu names shown):
|
|
371
377
|
- `sslscan`, `openssl` — TLS check (`apt install sslscan`; openssl is
|
|
@@ -450,8 +456,12 @@ process.
|
|
|
450
456
|
|
|
451
457
|
```bash
|
|
452
458
|
# Scan a list of targets (IPs, hostnames, and/or CIDR subnets, one per line)
|
|
453
|
-
# --output defaults to
|
|
454
|
-
#
|
|
459
|
+
# --output defaults to ~/Netscraper/results if omitted -- the same real
|
|
460
|
+
# folder every time, regardless of which directory you ran this from (you
|
|
461
|
+
# installed this with pip, so there's no "project folder" to default to
|
|
462
|
+
# the way running from a source checkout would have). With no --modes
|
|
463
|
+
# flag, you'll be prompted to pick which check(s) to run once the port
|
|
464
|
+
# scan finishes.
|
|
455
465
|
netscraper --targets targets.txt
|
|
456
466
|
|
|
457
467
|
# Scan a single target or a whole subnet on the command line
|
|
@@ -491,7 +501,7 @@ and — unless you passed `--modes` — asks which check(s) you actually want:
|
|
|
491
501
|
============================================================
|
|
492
502
|
[+] Target list loaded: 137 IP/host(es) to scan
|
|
493
503
|
[+] Scan type: LIVE
|
|
494
|
-
[+] Results folder: results/scan_20260907_143022_a1b2c3
|
|
504
|
+
[+] Results folder: /Users/kav1n/Netscraper/results/scan_20260907_143022_a1b2c3
|
|
495
505
|
============================================================
|
|
496
506
|
|
|
497
507
|
[+] Scanning required ports across 137 target(s) ...
|
|
@@ -537,12 +547,15 @@ the two checks with no fixed port of their own.
|
|
|
537
547
|
### Every run gets its own results folder
|
|
538
548
|
|
|
539
549
|
By default, each run writes into a uniquely-named subfolder under `--output`
|
|
540
|
-
(default root
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
550
|
+
(default root `~/Netscraper/results` — the same real folder on disk every
|
|
551
|
+
time, no matter which directory you happened to run `netscraper` from)
|
|
552
|
+
instead of a single fixed path — e.g.
|
|
553
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3` (a timestamp plus a
|
|
554
|
+
short random suffix, so two runs started in the same second never
|
|
555
|
+
collide). This means re-scanning the same targets later (a re-test after
|
|
556
|
+
remediation, a repeat engagement, etc.) never silently overwrites a
|
|
557
|
+
previous run's POCs and raw evidence — every run's evidence is kept, side
|
|
558
|
+
by side, under `~/Netscraper/results/`.
|
|
546
559
|
|
|
547
560
|
If you're scripting this and want full control over the exact output path
|
|
548
561
|
yourself (e.g. in CI), pass `--no-run-id` to write directly into `--output`
|
|
@@ -589,8 +602,8 @@ point of vuln-first scanning).
|
|
|
589
602
|
## Output layout
|
|
590
603
|
|
|
591
604
|
Everything below lives under this run's unique folder (e.g.
|
|
592
|
-
|
|
593
|
-
passed `--no-run-id`):
|
|
605
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3/`, or directly under
|
|
606
|
+
`--output` if you passed `--no-run-id`):
|
|
594
607
|
|
|
595
608
|
```
|
|
596
609
|
scan_<timestamp>_<id>/
|
|
@@ -6,15 +6,18 @@ ready to drop straight into a report appendix.
|
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
8
|
pip install netscraper
|
|
9
|
-
playwright install chromium # one-time, for POC screenshots
|
|
10
9
|
netscraper --targets targets.txt
|
|
11
10
|
```
|
|
12
11
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
That's it — no separate `playwright install chromium` step: the first time
|
|
13
|
+
the tool actually needs Chromium for a POC screenshot, it notices it isn't
|
|
14
|
+
downloaded yet and fetches it automatically (a one-time, ~150MB download;
|
|
15
|
+
every run after that is an instant no-op check). For authorized
|
|
16
|
+
penetration testing and security research only — see the legal banner the
|
|
17
|
+
tool itself prints on every run. See "Requirements" below for the external
|
|
18
|
+
command-line tools (`sslscan`, `nmap`, etc.) each check shells out to;
|
|
19
|
+
`pip install` alone gets you the `netscraper` command and its Python
|
|
20
|
+
dependencies, not those OS-level tools.
|
|
18
21
|
|
|
19
22
|
Runs **vuln-first, not host-first**: every check's required port (TLS's own
|
|
20
23
|
port, SSH/22, SMB/445, LDAP/389, RPC/445, MSSQL/1433, FTP/21, HTTP/80) gets
|
|
@@ -327,14 +330,17 @@ signing flagged for manual verification, both real, independent signals).
|
|
|
327
330
|
|
|
328
331
|
- Python 3.9+, on macOS or Linux (uses a Unix pty for ANSI-preserving
|
|
329
332
|
terminal capture where relevant; no Windows support).
|
|
330
|
-
- `pip install netscraper
|
|
331
|
-
`
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
`
|
|
337
|
-
the
|
|
333
|
+
- `pip install netscraper` (once) — pulls in `ssh-audit`, `impacket` (for
|
|
334
|
+
`rpcdump.py`), `dirsearch`, `playwright` (renders POC screenshots),
|
|
335
|
+
`ansi2html` (real captured terminal colors → matching HTML), and
|
|
336
|
+
`openpyxl` (writes `PT_Scan_Report.xlsx`) as dependencies automatically,
|
|
337
|
+
and puts the `netscraper` command on your PATH. No separate `playwright
|
|
338
|
+
install chromium` step needed — see `ensure_chromium_installed()` in
|
|
339
|
+
`core/poc_renderer.py`, which the `netscraper` command calls on startup
|
|
340
|
+
and which downloads Chromium the first time it notices it isn't there
|
|
341
|
+
yet. Running from a source checkout instead of the published package?
|
|
342
|
+
`pip install -e .` from the repo root does the same thing against your
|
|
343
|
+
local copy.
|
|
338
344
|
- Command-line tools this toolkit shells out to (install via your package
|
|
339
345
|
manager — Debian/Ubuntu names shown):
|
|
340
346
|
- `sslscan`, `openssl` — TLS check (`apt install sslscan`; openssl is
|
|
@@ -419,8 +425,12 @@ process.
|
|
|
419
425
|
|
|
420
426
|
```bash
|
|
421
427
|
# Scan a list of targets (IPs, hostnames, and/or CIDR subnets, one per line)
|
|
422
|
-
# --output defaults to
|
|
423
|
-
#
|
|
428
|
+
# --output defaults to ~/Netscraper/results if omitted -- the same real
|
|
429
|
+
# folder every time, regardless of which directory you ran this from (you
|
|
430
|
+
# installed this with pip, so there's no "project folder" to default to
|
|
431
|
+
# the way running from a source checkout would have). With no --modes
|
|
432
|
+
# flag, you'll be prompted to pick which check(s) to run once the port
|
|
433
|
+
# scan finishes.
|
|
424
434
|
netscraper --targets targets.txt
|
|
425
435
|
|
|
426
436
|
# Scan a single target or a whole subnet on the command line
|
|
@@ -460,7 +470,7 @@ and — unless you passed `--modes` — asks which check(s) you actually want:
|
|
|
460
470
|
============================================================
|
|
461
471
|
[+] Target list loaded: 137 IP/host(es) to scan
|
|
462
472
|
[+] Scan type: LIVE
|
|
463
|
-
[+] Results folder: results/scan_20260907_143022_a1b2c3
|
|
473
|
+
[+] Results folder: /Users/kav1n/Netscraper/results/scan_20260907_143022_a1b2c3
|
|
464
474
|
============================================================
|
|
465
475
|
|
|
466
476
|
[+] Scanning required ports across 137 target(s) ...
|
|
@@ -506,12 +516,15 @@ the two checks with no fixed port of their own.
|
|
|
506
516
|
### Every run gets its own results folder
|
|
507
517
|
|
|
508
518
|
By default, each run writes into a uniquely-named subfolder under `--output`
|
|
509
|
-
(default root
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
519
|
+
(default root `~/Netscraper/results` — the same real folder on disk every
|
|
520
|
+
time, no matter which directory you happened to run `netscraper` from)
|
|
521
|
+
instead of a single fixed path — e.g.
|
|
522
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3` (a timestamp plus a
|
|
523
|
+
short random suffix, so two runs started in the same second never
|
|
524
|
+
collide). This means re-scanning the same targets later (a re-test after
|
|
525
|
+
remediation, a repeat engagement, etc.) never silently overwrites a
|
|
526
|
+
previous run's POCs and raw evidence — every run's evidence is kept, side
|
|
527
|
+
by side, under `~/Netscraper/results/`.
|
|
515
528
|
|
|
516
529
|
If you're scripting this and want full control over the exact output path
|
|
517
530
|
yourself (e.g. in CI), pass `--no-run-id` to write directly into `--output`
|
|
@@ -558,8 +571,8 @@ point of vuln-first scanning).
|
|
|
558
571
|
## Output layout
|
|
559
572
|
|
|
560
573
|
Everything below lives under this run's unique folder (e.g.
|
|
561
|
-
|
|
562
|
-
passed `--no-run-id`):
|
|
574
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3/`, or directly under
|
|
575
|
+
`--output` if you passed `--no-run-id`):
|
|
563
576
|
|
|
564
577
|
```
|
|
565
578
|
scan_<timestamp>_<id>/
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "netscraper"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Automated network pentest check orchestration -- port-gates a target list, runs the relevant checks (TLS, SSH, SMB, LDAP, RPC, DNS, MSSQL, SNMP, FTP, HTTP, DIRSEARCH, CORS), and produces per-finding POC screenshots plus a styled Excel report."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -16,22 +16,23 @@ not lumped in with a "safe" result it didn't earn.
|
|
|
16
16
|
|
|
17
17
|
Usage:
|
|
18
18
|
# Live scan (needs the relevant tools installed + real network reach).
|
|
19
|
-
# --output defaults to
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
|
|
19
|
+
# --output defaults to ~/Netscraper/results if omitted -- the same real
|
|
20
|
+
# folder every time, regardless of which directory you ran this from.
|
|
21
|
+
# With no --modes flag, you'll be prompted to pick which check(s) to run
|
|
22
|
+
# once the port scan is done. After a scan finishes (or is interrupted
|
|
23
|
+
# with Ctrl-C), you're dropped back into the same Individual/Bulk/Exit
|
|
24
|
+
# main menu -- the tool only exits when you actually choose Exit there.
|
|
25
|
+
netscraper --targets targets.txt
|
|
25
26
|
|
|
26
27
|
# Single target on the command line
|
|
27
|
-
|
|
28
|
+
netscraper --target 192.168.1.10 --output ./results
|
|
28
29
|
|
|
29
30
|
# Skip the interactive picker and run specific check(s) non-interactively
|
|
30
|
-
|
|
31
|
+
netscraper --targets targets.txt --modes tls,ssh
|
|
31
32
|
|
|
32
33
|
# Replay previously-captured output instead of hitting the network
|
|
33
34
|
# (used for testing / reprocessing evidence; see fixtures/)
|
|
34
|
-
|
|
35
|
+
netscraper --targets targets_example.txt --output ./results --xml-dir ./fixtures
|
|
35
36
|
|
|
36
37
|
Output layout (per check):
|
|
37
38
|
<output>/SSL_Scan/Expired_Certificate/<host>.png (+ .txt)
|
|
@@ -72,20 +73,35 @@ from netscraper.core.port_gate import gate_targets, gate_targets_own_port
|
|
|
72
73
|
from netscraper.core.families import FAMILIES
|
|
73
74
|
from netscraper.core.concurrency import worker_count
|
|
74
75
|
from netscraper.core.scan_worker import run_one_target_check
|
|
76
|
+
from netscraper.core.poc_renderer import ensure_chromium_installed
|
|
75
77
|
from netscraper.core.excel_report import write_report
|
|
76
78
|
from netscraper.core.banner import clear_screen, print_banner, prompt_mode, prompt_scan_modes, prompt_port_list, ReturnToMainMenu
|
|
77
79
|
from netscraper.core.theme import title, accent, info, ok, bad, warn, err, muted, prompt, host as c_host
|
|
78
80
|
|
|
81
|
+
# A pip-installed CLI has no natural "project folder" the way a source
|
|
82
|
+
# checkout run with `python3 cli.py` from its own repo does -- `netscraper`
|
|
83
|
+
# can be invoked from literally any directory. Defaulting --output to a
|
|
84
|
+
# relative "./results" would then scatter evidence into whatever random
|
|
85
|
+
# folder happened to be the current directory at the time, which is
|
|
86
|
+
# surprising and easy to lose track of. Anchoring the default on the
|
|
87
|
+
# user's home directory instead means results always land in the same
|
|
88
|
+
# predictable place regardless of where `netscraper` was run from --
|
|
89
|
+
# ~/Netscraper/results on every OS this tool supports (macOS/Linux).
|
|
90
|
+
# --output still overrides this for anyone who wants a different location.
|
|
91
|
+
DEFAULT_OUTPUT_DIR = str(Path.home() / "Netscraper" / "results")
|
|
92
|
+
|
|
79
93
|
|
|
80
94
|
def parse_args(argv=None) -> argparse.Namespace:
|
|
81
95
|
p = argparse.ArgumentParser(description="NetScraper")
|
|
82
96
|
src = p.add_mutually_exclusive_group(required=False)
|
|
83
97
|
src.add_argument("--targets", help="Path to a targets file (IPs, hostnames, or CIDR subnets, one per line)")
|
|
84
98
|
src.add_argument("--target", help="A single IP, hostname, or CIDR subnet")
|
|
85
|
-
p.add_argument("--output", default=
|
|
86
|
-
help="Root output directory for POCs, raw evidence, and summary.csv
|
|
87
|
-
"
|
|
88
|
-
"
|
|
99
|
+
p.add_argument("--output", default=DEFAULT_OUTPUT_DIR,
|
|
100
|
+
help=f"Root output directory for POCs, raw evidence, and summary.csv "
|
|
101
|
+
f"(default: {DEFAULT_OUTPUT_DIR}, i.e. ~/Netscraper/results -- the same place "
|
|
102
|
+
f"regardless of which directory you ran `netscraper` from). Each run gets its own "
|
|
103
|
+
f"uniquely-named subfolder under this root (see --no-run-id) so repeated scans never "
|
|
104
|
+
f"overwrite each other's evidence.")
|
|
89
105
|
p.add_argument("--no-run-id", action="store_true",
|
|
90
106
|
help="Write directly into --output instead of a per-run unique subfolder (will overwrite "
|
|
91
107
|
"a previous run's evidence at that same path -- use this only for scripted/CI use "
|
|
@@ -591,6 +607,16 @@ def main(argv=None) -> int:
|
|
|
591
607
|
args = parse_args(argv)
|
|
592
608
|
interactive = not args.targets and not args.target
|
|
593
609
|
|
|
610
|
+
# Runs exactly once per process, before anything else -- makes `pip
|
|
611
|
+
# install netscraper` followed immediately by running this command
|
|
612
|
+
# just work, with no separate manual `playwright install chromium`
|
|
613
|
+
# step, by downloading it automatically the first time it's actually
|
|
614
|
+
# missing (a fast no-op every run after that). Deliberately done here,
|
|
615
|
+
# in the main process, before the interactive/scripted branches below
|
|
616
|
+
# ever create a worker pool -- see ensure_chromium_installed()'s
|
|
617
|
+
# docstring for why that ordering matters.
|
|
618
|
+
ensure_chromium_installed()
|
|
619
|
+
|
|
594
620
|
if not interactive:
|
|
595
621
|
# Scripted/automated use (--targets or --target given up front):
|
|
596
622
|
# run exactly once and exit, same as before -- no menu loop, since
|
|
@@ -13,6 +13,8 @@ from __future__ import annotations
|
|
|
13
13
|
import atexit
|
|
14
14
|
import html
|
|
15
15
|
import re
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
16
18
|
from contextlib import contextmanager
|
|
17
19
|
from pathlib import Path
|
|
18
20
|
from typing import List, Optional, Pattern
|
|
@@ -44,6 +46,19 @@ _SCREENSHOT_TIMEOUT_MS = 15_000
|
|
|
44
46
|
# blue for exactly this reason.
|
|
45
47
|
_ANSI_CONVERTER = Ansi2HTMLConverter(inline=True, dark_bg=True, scheme="osx")
|
|
46
48
|
|
|
49
|
+
# Used ONLY to decide which lines count as a highlight match (see
|
|
50
|
+
# _build_body_html below) -- never used to alter what's actually rendered.
|
|
51
|
+
# Some tools (sslscan included) color just PART of a string differently from
|
|
52
|
+
# the rest of it -- e.g. the "-SHA" suffix of a cipher name in a different
|
|
53
|
+
# shade than the rest of the name -- which plants a raw ANSI escape code
|
|
54
|
+
# *inside* what should be one contiguous run of characters. A regex like
|
|
55
|
+
# \bECDHE-ECDSA-AES256-SHA\b can never match that string while the escape
|
|
56
|
+
# code is still sitting in the middle of it, even though the *visible* text
|
|
57
|
+
# is exactly what the pattern is looking for. Stripping codes before
|
|
58
|
+
# matching (and only before matching) fixes that without touching the HTML
|
|
59
|
+
# color rendering itself, which still runs on the original, code-intact text.
|
|
60
|
+
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
|
61
|
+
|
|
47
62
|
# How much context to keep around each highlighted (matched) line when a POC
|
|
48
63
|
# gets cropped down to "just the relevant section" for the report. This only
|
|
49
64
|
# ever kicks in when there's at least one real regex hit on a real line --
|
|
@@ -189,9 +204,12 @@ def _build_body_html(raw_text: str, highlight_regex: Optional[Pattern]) -> str:
|
|
|
189
204
|
if len(html_lines) != len(raw_lines):
|
|
190
205
|
html_lines = [html.escape(line) for line in raw_lines]
|
|
191
206
|
|
|
207
|
+
# Match against the ANSI-stripped text, not raw_line itself -- see the
|
|
208
|
+
# _ANSI_RE comment above for why a code embedded mid-string can otherwise
|
|
209
|
+
# hide an obvious, visible match from the regex entirely.
|
|
192
210
|
hit_indices = [
|
|
193
211
|
i for i, raw_line in enumerate(raw_lines)
|
|
194
|
-
if highlight_regex and highlight_regex.search(raw_line)
|
|
212
|
+
if highlight_regex and highlight_regex.search(_ANSI_RE.sub("", raw_line))
|
|
195
213
|
]
|
|
196
214
|
keep_ranges = _compute_keep_ranges(
|
|
197
215
|
len(raw_lines), hit_indices, _CONTEXT_BEFORE, _CONTEXT_AFTER
|
|
@@ -236,6 +254,58 @@ def _launch_browser(p):
|
|
|
236
254
|
) from exc
|
|
237
255
|
|
|
238
256
|
|
|
257
|
+
def ensure_chromium_installed() -> None:
|
|
258
|
+
"""
|
|
259
|
+
Makes `pip install netscraper` followed immediately by `netscraper
|
|
260
|
+
--targets ...` just work, with no separate manual `playwright install
|
|
261
|
+
chromium` step -- checks whether Playwright's Chromium build is
|
|
262
|
+
actually on disk yet and, if not, downloads it automatically (once)
|
|
263
|
+
before any real scanning starts.
|
|
264
|
+
|
|
265
|
+
Cheap on every run after the first: `p.chromium.executable_path` is
|
|
266
|
+
just Playwright telling us where it WOULD look for the browser, not
|
|
267
|
+
launching anything, so once it's been downloaded this is a fast path
|
|
268
|
+
check, not a re-download.
|
|
269
|
+
|
|
270
|
+
Call this exactly ONCE, in the main process, before any worker pool or
|
|
271
|
+
scan loop starts -- worker processes forked afterward inherit the
|
|
272
|
+
already-downloaded browser from disk and never hit this themselves,
|
|
273
|
+
which matters: without a single up-front check like this, several
|
|
274
|
+
worker processes could each discover Chromium missing on a brand-new
|
|
275
|
+
machine's very first run and race to download it into the same cache
|
|
276
|
+
directory at once.
|
|
277
|
+
"""
|
|
278
|
+
if Path(_SANDBOX_CHROMIUM_PATH).exists():
|
|
279
|
+
return # sandbox's own pre-fetched Chromium -- nothing to install
|
|
280
|
+
|
|
281
|
+
try:
|
|
282
|
+
with sync_playwright() as p:
|
|
283
|
+
exe_path = p.chromium.executable_path
|
|
284
|
+
except Exception:
|
|
285
|
+
# Can't even ask Playwright where it expects the browser (a truly
|
|
286
|
+
# broken Playwright install) -- don't try to guess further here;
|
|
287
|
+
# the real render call later will surface a clear, actionable
|
|
288
|
+
# error instead (see _launch_browser above).
|
|
289
|
+
return
|
|
290
|
+
|
|
291
|
+
if exe_path and Path(exe_path).exists():
|
|
292
|
+
return # already installed
|
|
293
|
+
|
|
294
|
+
print("[*] Chromium not found -- downloading it now for POC screenshots "
|
|
295
|
+
"(one-time, ~150MB, needs network access) ...")
|
|
296
|
+
try:
|
|
297
|
+
result = subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"])
|
|
298
|
+
except Exception as exc:
|
|
299
|
+
print(f"[!] Could not run the automatic Chromium download ({exc}). "
|
|
300
|
+
f"Run `playwright install chromium` yourself, then try again.")
|
|
301
|
+
return
|
|
302
|
+
if result.returncode != 0:
|
|
303
|
+
print("[!] Automatic Chromium download did not finish successfully. "
|
|
304
|
+
"Run `playwright install chromium` yourself, then try again.")
|
|
305
|
+
else:
|
|
306
|
+
print("[+] Chromium downloaded.")
|
|
307
|
+
|
|
308
|
+
|
|
239
309
|
@contextmanager
|
|
240
310
|
def browser_session():
|
|
241
311
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: netscraper
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Automated network pentest check orchestration -- port-gates a target list, runs the relevant checks (TLS, SSH, SMB, LDAP, RPC, DNS, MSSQL, SNMP, FTP, HTTP, DIRSEARCH, CORS), and produces per-finding POC screenshots plus a styled Excel report.
|
|
5
5
|
Author: Wati Cyber
|
|
6
6
|
License: MIT
|
|
@@ -37,15 +37,18 @@ ready to drop straight into a report appendix.
|
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
39
|
pip install netscraper
|
|
40
|
-
playwright install chromium # one-time, for POC screenshots
|
|
41
40
|
netscraper --targets targets.txt
|
|
42
41
|
```
|
|
43
42
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
That's it — no separate `playwright install chromium` step: the first time
|
|
44
|
+
the tool actually needs Chromium for a POC screenshot, it notices it isn't
|
|
45
|
+
downloaded yet and fetches it automatically (a one-time, ~150MB download;
|
|
46
|
+
every run after that is an instant no-op check). For authorized
|
|
47
|
+
penetration testing and security research only — see the legal banner the
|
|
48
|
+
tool itself prints on every run. See "Requirements" below for the external
|
|
49
|
+
command-line tools (`sslscan`, `nmap`, etc.) each check shells out to;
|
|
50
|
+
`pip install` alone gets you the `netscraper` command and its Python
|
|
51
|
+
dependencies, not those OS-level tools.
|
|
49
52
|
|
|
50
53
|
Runs **vuln-first, not host-first**: every check's required port (TLS's own
|
|
51
54
|
port, SSH/22, SMB/445, LDAP/389, RPC/445, MSSQL/1433, FTP/21, HTTP/80) gets
|
|
@@ -358,14 +361,17 @@ signing flagged for manual verification, both real, independent signals).
|
|
|
358
361
|
|
|
359
362
|
- Python 3.9+, on macOS or Linux (uses a Unix pty for ANSI-preserving
|
|
360
363
|
terminal capture where relevant; no Windows support).
|
|
361
|
-
- `pip install netscraper
|
|
362
|
-
`
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
`
|
|
368
|
-
the
|
|
364
|
+
- `pip install netscraper` (once) — pulls in `ssh-audit`, `impacket` (for
|
|
365
|
+
`rpcdump.py`), `dirsearch`, `playwright` (renders POC screenshots),
|
|
366
|
+
`ansi2html` (real captured terminal colors → matching HTML), and
|
|
367
|
+
`openpyxl` (writes `PT_Scan_Report.xlsx`) as dependencies automatically,
|
|
368
|
+
and puts the `netscraper` command on your PATH. No separate `playwright
|
|
369
|
+
install chromium` step needed — see `ensure_chromium_installed()` in
|
|
370
|
+
`core/poc_renderer.py`, which the `netscraper` command calls on startup
|
|
371
|
+
and which downloads Chromium the first time it notices it isn't there
|
|
372
|
+
yet. Running from a source checkout instead of the published package?
|
|
373
|
+
`pip install -e .` from the repo root does the same thing against your
|
|
374
|
+
local copy.
|
|
369
375
|
- Command-line tools this toolkit shells out to (install via your package
|
|
370
376
|
manager — Debian/Ubuntu names shown):
|
|
371
377
|
- `sslscan`, `openssl` — TLS check (`apt install sslscan`; openssl is
|
|
@@ -450,8 +456,12 @@ process.
|
|
|
450
456
|
|
|
451
457
|
```bash
|
|
452
458
|
# Scan a list of targets (IPs, hostnames, and/or CIDR subnets, one per line)
|
|
453
|
-
# --output defaults to
|
|
454
|
-
#
|
|
459
|
+
# --output defaults to ~/Netscraper/results if omitted -- the same real
|
|
460
|
+
# folder every time, regardless of which directory you ran this from (you
|
|
461
|
+
# installed this with pip, so there's no "project folder" to default to
|
|
462
|
+
# the way running from a source checkout would have). With no --modes
|
|
463
|
+
# flag, you'll be prompted to pick which check(s) to run once the port
|
|
464
|
+
# scan finishes.
|
|
455
465
|
netscraper --targets targets.txt
|
|
456
466
|
|
|
457
467
|
# Scan a single target or a whole subnet on the command line
|
|
@@ -491,7 +501,7 @@ and — unless you passed `--modes` — asks which check(s) you actually want:
|
|
|
491
501
|
============================================================
|
|
492
502
|
[+] Target list loaded: 137 IP/host(es) to scan
|
|
493
503
|
[+] Scan type: LIVE
|
|
494
|
-
[+] Results folder: results/scan_20260907_143022_a1b2c3
|
|
504
|
+
[+] Results folder: /Users/kav1n/Netscraper/results/scan_20260907_143022_a1b2c3
|
|
495
505
|
============================================================
|
|
496
506
|
|
|
497
507
|
[+] Scanning required ports across 137 target(s) ...
|
|
@@ -537,12 +547,15 @@ the two checks with no fixed port of their own.
|
|
|
537
547
|
### Every run gets its own results folder
|
|
538
548
|
|
|
539
549
|
By default, each run writes into a uniquely-named subfolder under `--output`
|
|
540
|
-
(default root
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
550
|
+
(default root `~/Netscraper/results` — the same real folder on disk every
|
|
551
|
+
time, no matter which directory you happened to run `netscraper` from)
|
|
552
|
+
instead of a single fixed path — e.g.
|
|
553
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3` (a timestamp plus a
|
|
554
|
+
short random suffix, so two runs started in the same second never
|
|
555
|
+
collide). This means re-scanning the same targets later (a re-test after
|
|
556
|
+
remediation, a repeat engagement, etc.) never silently overwrites a
|
|
557
|
+
previous run's POCs and raw evidence — every run's evidence is kept, side
|
|
558
|
+
by side, under `~/Netscraper/results/`.
|
|
546
559
|
|
|
547
560
|
If you're scripting this and want full control over the exact output path
|
|
548
561
|
yourself (e.g. in CI), pass `--no-run-id` to write directly into `--output`
|
|
@@ -589,8 +602,8 @@ point of vuln-first scanning).
|
|
|
589
602
|
## Output layout
|
|
590
603
|
|
|
591
604
|
Everything below lives under this run's unique folder (e.g.
|
|
592
|
-
|
|
593
|
-
passed `--no-run-id`):
|
|
605
|
+
`~/Netscraper/results/scan_20260907_143022_a1b2c3/`, or directly under
|
|
606
|
+
`--output` if you passed `--no-run-id`):
|
|
594
607
|
|
|
595
608
|
```
|
|
596
609
|
scan_<timestamp>_<id>/
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|