ipmg 2.1.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ipmg-2.1.0/src/ipmg.egg-info → ipmg-2.2.0}/PKG-INFO +36 -10
- {ipmg-2.1.0 → ipmg-2.2.0}/README.md +35 -9
- {ipmg-2.1.0 → ipmg-2.2.0}/pyproject.toml +1 -1
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/__init__.py +1 -1
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/cli/parser.py +12 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/engine.py +47 -14
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/portscan.py +19 -10
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/infrastructure/incremental.py +213 -5
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/ui.py +2 -1
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/services/scan_service.py +61 -9
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/utils/helpers.py +5 -1
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/app.py +75 -9
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/manager.py +24 -2
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/server.py +37 -5
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/index.html +2 -2
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/js/api.js +63 -2
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/js/app.js +2 -2
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/js/views.js +25 -8
- {ipmg-2.1.0 → ipmg-2.2.0/src/ipmg.egg-info}/PKG-INFO +36 -10
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg.egg-info/SOURCES.txt +1 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_engine.py +54 -1
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_incremental.py +163 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_parser.py +10 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_utils.py +12 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_web_api.py +95 -3
- ipmg-2.2.0/tests/test_web_manager.py +36 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_web_server.py +51 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/LICENSE +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/setup.cfg +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/__main__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/cli/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/cli/commands.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/diff.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/discovery.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/ping.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/core/security.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/exceptions.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/infrastructure/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/infrastructure/database.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/infrastructure/file_io.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/diff_report.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/frames.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/live.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/reporting/summary.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/services/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/services/history_service.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/utils/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/__init__.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/db.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/css/app.css +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/js/charts.js +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg/web/static/js/demo.js +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg.egg-info/dependency_links.txt +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg.egg-info/entry_points.txt +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg.egg-info/requires.txt +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/src/ipmg.egg-info/top_level.txt +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_commands.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_database_history.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_diff.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_diff_report.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_discover.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_file_io.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_history_service.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_live.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_ping.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_ping_command.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_portscan.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_scan_service.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_ui.py +0 -0
- {ipmg-2.1.0 → ipmg-2.2.0}/tests/test_web_db.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ipmg
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: IP Management & Ping Monitoring CLI Tool
|
|
5
5
|
Author: Sameer Alam
|
|
6
6
|
Maintainer-email: Sameer Alam <sameeralam3127@gmail.com>
|
|
@@ -436,6 +436,12 @@ ipmg web # starts http://127.0.0.1:8080 and opens your browser
|
|
|
436
436
|
|
|
437
437
|
`ipmg --web` does the same thing, so whichever one you reach for first works.
|
|
438
438
|
|
|
439
|
+
Each start creates a new access token. The browser opens with it already in
|
|
440
|
+
the link, and IPMG Web prints that link (`http://127.0.0.1:8080/#token=…`)
|
|
441
|
+
in the terminal. If you open IPMG Web in another browser, or after a
|
|
442
|
+
restart, use the link from the terminal. To keep the same token across
|
|
443
|
+
restarts, for example behind a reverse proxy, set `IPMG_WEB_TOKEN`.
|
|
444
|
+
|
|
439
445
|
It runs fully offline — every stylesheet and script is bundled with the
|
|
440
446
|
package, nothing is loaded from a CDN. It gives you:
|
|
441
447
|
|
|
@@ -465,12 +471,13 @@ it from your workstation with an SSH tunnel:
|
|
|
465
471
|
|
|
466
472
|
```bash
|
|
467
473
|
ssh -L 8080:127.0.0.1:8080 user@server
|
|
468
|
-
# then open http://127.0.0.1:8080 locally
|
|
474
|
+
# then open the link the server printed (http://127.0.0.1:8080/#token=…) locally
|
|
469
475
|
```
|
|
470
476
|
|
|
471
|
-
Alternatively, bind to all interfaces with `--host 0.0.0.0
|
|
472
|
-
|
|
473
|
-
|
|
477
|
+
Alternatively, bind to all interfaces with `--host 0.0.0.0`. Every request
|
|
478
|
+
still needs the access token. The traffic, token included, is plain HTTP,
|
|
479
|
+
so on anything but a trusted network put a reverse proxy with TLS in front
|
|
480
|
+
of it (see [Security](#security)).
|
|
474
481
|
|
|
475
482
|
---
|
|
476
483
|
|
|
@@ -535,6 +542,21 @@ default). A finished scan overwrites those files with the complete report, so
|
|
|
535
542
|
the file names and contents are the same as they always were. Use
|
|
536
543
|
`--no-incremental` to go back to writing only at the end.
|
|
537
544
|
|
|
545
|
+
**Pick up where an interrupted scan stopped.** `--resume` reads the hosts the
|
|
546
|
+
partial report already holds, scans only the rest, and finishes that same
|
|
547
|
+
report — same file names, same batch timestamp:
|
|
548
|
+
|
|
549
|
+
```bash
|
|
550
|
+
ipmg --input 10.0.0.0/16 --formats jsonl xlsx # Ctrl+C partway through
|
|
551
|
+
ipmg --input 10.0.0.0/16 --formats jsonl xlsx --resume
|
|
552
|
+
ipmg --input 10.0.0.0/16 --resume results_20260917_120000.csv
|
|
553
|
+
```
|
|
554
|
+
|
|
555
|
+
Without a path, `--resume` takes the newest report named after `--output`,
|
|
556
|
+
preferring `jsonl` or `csv` (current to the last host) over `json` or `xlsx`
|
|
557
|
+
(current to the last autosave). Hosts dropped from the target list since are
|
|
558
|
+
left out of the finished report.
|
|
559
|
+
|
|
538
560
|
**Open ports.** `Open Ports` is only populated when `--scan-ports` is set: for
|
|
539
561
|
each host that answers, IPMG probes a list of common TCP ports (SSH, HTTP,
|
|
540
562
|
HTTPS, RDP, SMB, FTP, SMTP, DNS, MSSQL, MySQL, PostgreSQL by default)
|
|
@@ -565,6 +587,7 @@ them — so piping IPMG into a file or a log gives you clean text.
|
|
|
565
587
|
| `--formats` | `xlsx` | One or more of `xlsx`, `csv`, `json`, `jsonl`, `md` |
|
|
566
588
|
| `--no-incremental` | off | Only write the report once the scan has finished |
|
|
567
589
|
| `--autosave` | `30` | How often a running scan re-saves `xlsx`, `json`, and `md` |
|
|
590
|
+
| `--resume` | off | Finish an interrupted scan from its partial report (newest one, or the path given) |
|
|
568
591
|
|
|
569
592
|
**Speed and accuracy**
|
|
570
593
|
|
|
@@ -612,15 +635,18 @@ itself is hardened accordingly:
|
|
|
612
635
|
validated as an IP address first
|
|
613
636
|
- IPMG Web binds to `127.0.0.1` by default and serves everything
|
|
614
637
|
locally — no CDN assets, no outbound requests
|
|
615
|
-
-
|
|
616
|
-
|
|
638
|
+
- Every API request and WebSocket needs the access token created when
|
|
639
|
+
IPMG Web starts, so other users on the machine and web pages you happen
|
|
640
|
+
to visit cannot start scans or read your results
|
|
641
|
+
- WebSocket connections are also origin-checked, and a client that stops
|
|
642
|
+
reading live updates is disconnected rather than buffered without limit
|
|
617
643
|
- Uploads are capped at 5 MB and one scan expands to at most 65,536 hosts,
|
|
618
644
|
so a bad input file cannot exhaust memory
|
|
619
645
|
- All database access uses parameterized SQL
|
|
620
646
|
|
|
621
|
-
If you bind to a non-local address with `--host`,
|
|
622
|
-
|
|
623
|
-
|
|
647
|
+
If you bind to a non-local address with `--host`, the token still guards the
|
|
648
|
+
API. It travels over plain HTTP, though, so use an SSH tunnel or a reverse
|
|
649
|
+
proxy with TLS on any network you don't trust.
|
|
624
650
|
|
|
625
651
|
Found a vulnerability? See [SECURITY.md](SECURITY.md) for how to report it.
|
|
626
652
|
|
|
@@ -389,6 +389,12 @@ ipmg web # starts http://127.0.0.1:8080 and opens your browser
|
|
|
389
389
|
|
|
390
390
|
`ipmg --web` does the same thing, so whichever one you reach for first works.
|
|
391
391
|
|
|
392
|
+
Each start creates a new access token. The browser opens with it already in
|
|
393
|
+
the link, and IPMG Web prints that link (`http://127.0.0.1:8080/#token=…`)
|
|
394
|
+
in the terminal. If you open IPMG Web in another browser, or after a
|
|
395
|
+
restart, use the link from the terminal. To keep the same token across
|
|
396
|
+
restarts, for example behind a reverse proxy, set `IPMG_WEB_TOKEN`.
|
|
397
|
+
|
|
392
398
|
It runs fully offline — every stylesheet and script is bundled with the
|
|
393
399
|
package, nothing is loaded from a CDN. It gives you:
|
|
394
400
|
|
|
@@ -418,12 +424,13 @@ it from your workstation with an SSH tunnel:
|
|
|
418
424
|
|
|
419
425
|
```bash
|
|
420
426
|
ssh -L 8080:127.0.0.1:8080 user@server
|
|
421
|
-
# then open http://127.0.0.1:8080 locally
|
|
427
|
+
# then open the link the server printed (http://127.0.0.1:8080/#token=…) locally
|
|
422
428
|
```
|
|
423
429
|
|
|
424
|
-
Alternatively, bind to all interfaces with `--host 0.0.0.0
|
|
425
|
-
|
|
426
|
-
|
|
430
|
+
Alternatively, bind to all interfaces with `--host 0.0.0.0`. Every request
|
|
431
|
+
still needs the access token. The traffic, token included, is plain HTTP,
|
|
432
|
+
so on anything but a trusted network put a reverse proxy with TLS in front
|
|
433
|
+
of it (see [Security](#security)).
|
|
427
434
|
|
|
428
435
|
---
|
|
429
436
|
|
|
@@ -488,6 +495,21 @@ default). A finished scan overwrites those files with the complete report, so
|
|
|
488
495
|
the file names and contents are the same as they always were. Use
|
|
489
496
|
`--no-incremental` to go back to writing only at the end.
|
|
490
497
|
|
|
498
|
+
**Pick up where an interrupted scan stopped.** `--resume` reads the hosts the
|
|
499
|
+
partial report already holds, scans only the rest, and finishes that same
|
|
500
|
+
report — same file names, same batch timestamp:
|
|
501
|
+
|
|
502
|
+
```bash
|
|
503
|
+
ipmg --input 10.0.0.0/16 --formats jsonl xlsx # Ctrl+C partway through
|
|
504
|
+
ipmg --input 10.0.0.0/16 --formats jsonl xlsx --resume
|
|
505
|
+
ipmg --input 10.0.0.0/16 --resume results_20260917_120000.csv
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
Without a path, `--resume` takes the newest report named after `--output`,
|
|
509
|
+
preferring `jsonl` or `csv` (current to the last host) over `json` or `xlsx`
|
|
510
|
+
(current to the last autosave). Hosts dropped from the target list since are
|
|
511
|
+
left out of the finished report.
|
|
512
|
+
|
|
491
513
|
**Open ports.** `Open Ports` is only populated when `--scan-ports` is set: for
|
|
492
514
|
each host that answers, IPMG probes a list of common TCP ports (SSH, HTTP,
|
|
493
515
|
HTTPS, RDP, SMB, FTP, SMTP, DNS, MSSQL, MySQL, PostgreSQL by default)
|
|
@@ -518,6 +540,7 @@ them — so piping IPMG into a file or a log gives you clean text.
|
|
|
518
540
|
| `--formats` | `xlsx` | One or more of `xlsx`, `csv`, `json`, `jsonl`, `md` |
|
|
519
541
|
| `--no-incremental` | off | Only write the report once the scan has finished |
|
|
520
542
|
| `--autosave` | `30` | How often a running scan re-saves `xlsx`, `json`, and `md` |
|
|
543
|
+
| `--resume` | off | Finish an interrupted scan from its partial report (newest one, or the path given) |
|
|
521
544
|
|
|
522
545
|
**Speed and accuracy**
|
|
523
546
|
|
|
@@ -565,15 +588,18 @@ itself is hardened accordingly:
|
|
|
565
588
|
validated as an IP address first
|
|
566
589
|
- IPMG Web binds to `127.0.0.1` by default and serves everything
|
|
567
590
|
locally — no CDN assets, no outbound requests
|
|
568
|
-
-
|
|
569
|
-
|
|
591
|
+
- Every API request and WebSocket needs the access token created when
|
|
592
|
+
IPMG Web starts, so other users on the machine and web pages you happen
|
|
593
|
+
to visit cannot start scans or read your results
|
|
594
|
+
- WebSocket connections are also origin-checked, and a client that stops
|
|
595
|
+
reading live updates is disconnected rather than buffered without limit
|
|
570
596
|
- Uploads are capped at 5 MB and one scan expands to at most 65,536 hosts,
|
|
571
597
|
so a bad input file cannot exhaust memory
|
|
572
598
|
- All database access uses parameterized SQL
|
|
573
599
|
|
|
574
|
-
If you bind to a non-local address with `--host`,
|
|
575
|
-
|
|
576
|
-
|
|
600
|
+
If you bind to a non-local address with `--host`, the token still guards the
|
|
601
|
+
API. It travels over plain HTTP, though, so use an SSH tunnel or a reverse
|
|
602
|
+
proxy with TLS on any network you don't trust.
|
|
577
603
|
|
|
578
604
|
Found a vulnerability? See [SECURITY.md](SECURITY.md) for how to report it.
|
|
579
605
|
|
|
@@ -13,7 +13,7 @@ build-backend = "setuptools.build_meta"
|
|
|
13
13
|
|
|
14
14
|
[project]
|
|
15
15
|
name = "ipmg"
|
|
16
|
-
version = "2.
|
|
16
|
+
version = "2.2.0" # Managed automatically by semantic-release
|
|
17
17
|
description = "IP Management & Ping Monitoring CLI Tool"
|
|
18
18
|
readme = "README.md"
|
|
19
19
|
requires-python = ">=3.9"
|
|
@@ -219,6 +219,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
219
219
|
"csv and jsonl are written per host regardless."
|
|
220
220
|
),
|
|
221
221
|
)
|
|
222
|
+
reports.add_argument(
|
|
223
|
+
"--resume",
|
|
224
|
+
nargs="?",
|
|
225
|
+
const="",
|
|
226
|
+
default=None,
|
|
227
|
+
metavar="REPORT",
|
|
228
|
+
help=(
|
|
229
|
+
"Finish an interrupted scan: skip the hosts its report already holds and "
|
|
230
|
+
"complete that report. REPORT is its jsonl, csv, json, or xlsx file "
|
|
231
|
+
"(default: the newest report named after --output)."
|
|
232
|
+
),
|
|
233
|
+
)
|
|
222
234
|
|
|
223
235
|
live = parser.add_argument_group("live output")
|
|
224
236
|
live.add_argument(
|
|
@@ -11,6 +11,11 @@ from ipmg.core.portscan import DEFAULT_PORTS, scan_ports
|
|
|
11
11
|
from ipmg.exceptions import PingError
|
|
12
12
|
from ipmg.utils.helpers import HostnameCache, clamp_int
|
|
13
13
|
|
|
14
|
+
#: Upper bound on TCP connect probes in flight across a whole scan. Every host
|
|
15
|
+
#: worker shares this one pool, so --scan-ports cannot create
|
|
16
|
+
#: threads x ports probe threads.
|
|
17
|
+
MAX_PORT_PROBE_WORKERS = 128
|
|
18
|
+
|
|
14
19
|
|
|
15
20
|
@dataclass(frozen=True)
|
|
16
21
|
class ScanConfig:
|
|
@@ -65,32 +70,27 @@ def execute_scan(
|
|
|
65
70
|
cache = HostnameCache(config.dns_cache_ttl) if config.resolve else None
|
|
66
71
|
results: List[HostResult] = []
|
|
67
72
|
|
|
73
|
+
port_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
|
74
|
+
if config.scan_ports and config.ports:
|
|
75
|
+
port_executor = concurrent.futures.ThreadPoolExecutor(
|
|
76
|
+
max_workers=min(MAX_PORT_PROBE_WORKERS, config.threads * len(config.ports)),
|
|
77
|
+
thread_name_prefix="ipmg-port",
|
|
78
|
+
)
|
|
79
|
+
|
|
68
80
|
executor = concurrent.futures.ThreadPoolExecutor(max_workers=config.threads)
|
|
69
81
|
try:
|
|
70
|
-
futures =
|
|
82
|
+
futures = [executor.submit(_probe_host, ip, config, cache, port_executor) for ip in ips]
|
|
71
83
|
|
|
72
84
|
for future in concurrent.futures.as_completed(futures):
|
|
73
85
|
if should_stop is not None and should_stop():
|
|
74
86
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
75
87
|
break
|
|
76
88
|
|
|
77
|
-
ip = futures[future]
|
|
78
89
|
try:
|
|
79
|
-
|
|
90
|
+
result = future.result()
|
|
80
91
|
except PingError:
|
|
81
92
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
82
93
|
raise
|
|
83
|
-
except Exception:
|
|
84
|
-
status, latency = "Error", None
|
|
85
|
-
|
|
86
|
-
hostname = cache.resolve(ip) if cache else ""
|
|
87
|
-
open_ports: Tuple[int, ...] = ()
|
|
88
|
-
if config.scan_ports and status == "Active":
|
|
89
|
-
open_ports = tuple(scan_ports(ip, config.ports, config.port_timeout))
|
|
90
|
-
|
|
91
|
-
result = HostResult(
|
|
92
|
-
ip=ip, status=status, latency=latency, hostname=hostname, open_ports=open_ports
|
|
93
|
-
)
|
|
94
94
|
results.append(result)
|
|
95
95
|
|
|
96
96
|
if on_result is not None:
|
|
@@ -103,5 +103,38 @@ def execute_scan(
|
|
|
103
103
|
raise
|
|
104
104
|
finally:
|
|
105
105
|
executor.shutdown(wait=True)
|
|
106
|
+
if port_executor is not None:
|
|
107
|
+
port_executor.shutdown(wait=True)
|
|
106
108
|
|
|
107
109
|
return results
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _probe_host(
|
|
113
|
+
ip: str,
|
|
114
|
+
config: ScanConfig,
|
|
115
|
+
cache: Optional[HostnameCache],
|
|
116
|
+
port_executor: Optional[concurrent.futures.Executor],
|
|
117
|
+
) -> HostResult:
|
|
118
|
+
"""Ping one host, then resolve its name and probe its ports.
|
|
119
|
+
|
|
120
|
+
Runs on a pool worker, so reverse DNS is spread across ``config.threads``
|
|
121
|
+
workers instead of running one host at a time on the thread that collects
|
|
122
|
+
results. Port probes go to the scan's shared, bounded ``port_executor``.
|
|
123
|
+
"""
|
|
124
|
+
try:
|
|
125
|
+
status, latency = ping_ip(ip, config.timeout, config.count)
|
|
126
|
+
except PingError:
|
|
127
|
+
raise
|
|
128
|
+
except Exception:
|
|
129
|
+
status, latency = "Error", None
|
|
130
|
+
|
|
131
|
+
hostname = cache.resolve(ip) if cache else ""
|
|
132
|
+
open_ports: Tuple[int, ...] = ()
|
|
133
|
+
if config.scan_ports and status == "Active":
|
|
134
|
+
open_ports = tuple(
|
|
135
|
+
scan_ports(ip, config.ports, config.port_timeout, executor=port_executor)
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
return HostResult(
|
|
139
|
+
ip=ip, status=status, latency=latency, hostname=hostname, open_ports=open_ports
|
|
140
|
+
)
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import concurrent.futures
|
|
6
6
|
import socket
|
|
7
|
-
from typing import Iterable, List, Tuple
|
|
7
|
+
from typing import Iterable, List, Optional, Tuple
|
|
8
8
|
|
|
9
9
|
#: Common services worth a quick TCP connect once a host is known to be up.
|
|
10
10
|
DEFAULT_PORTS: Tuple[int, ...] = (21, 22, 25, 53, 80, 443, 445, 1433, 3306, 3389, 5432)
|
|
@@ -65,18 +65,27 @@ def parse_port_list(value: str) -> Tuple[int, ...]:
|
|
|
65
65
|
return tuple(ports)
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
def scan_ports(
|
|
69
|
-
|
|
68
|
+
def scan_ports(
|
|
69
|
+
ip: str,
|
|
70
|
+
ports: Iterable[int],
|
|
71
|
+
timeout: float = 1.0,
|
|
72
|
+
executor: Optional[concurrent.futures.Executor] = None,
|
|
73
|
+
) -> List[int]:
|
|
74
|
+
"""Probe ``ports`` on ``ip`` concurrently; return the ones that accepted a connection.
|
|
75
|
+
|
|
76
|
+
Pass ``executor`` to share one bounded pool across many hosts; otherwise a
|
|
77
|
+
pool with one worker per port is created for this call.
|
|
78
|
+
"""
|
|
70
79
|
ports = list(ports)
|
|
71
80
|
if not ports:
|
|
72
81
|
return []
|
|
73
82
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
futures[future]
|
|
78
|
-
for future in concurrent.futures.as_completed(futures)
|
|
79
|
-
if future.result()
|
|
80
|
-
]
|
|
83
|
+
if executor is None:
|
|
84
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=len(ports)) as own:
|
|
85
|
+
return scan_ports(ip, ports, timeout, executor=own)
|
|
81
86
|
|
|
87
|
+
futures = {executor.submit(_probe, ip, port, timeout): port for port in ports}
|
|
88
|
+
open_ports = [
|
|
89
|
+
futures[future] for future in concurrent.futures.as_completed(futures) if future.result()
|
|
90
|
+
]
|
|
82
91
|
return sorted(open_ports)
|
|
@@ -17,23 +17,32 @@ writes at the end of the pass, which is why the scan shares its timestamp with
|
|
|
17
17
|
the writer. A completed scan overwrites every snapshot with the canonical
|
|
18
18
|
report, so incremental writing changes what survives an interruption without
|
|
19
19
|
changing what a finished scan produces.
|
|
20
|
+
|
|
21
|
+
That partial report is also where an interrupted scan picks up again:
|
|
22
|
+
:func:`load_partial_report` reads the hosts it already holds, and ``--resume``
|
|
23
|
+
scans only the rest, writing to the same files under the same timestamp.
|
|
20
24
|
"""
|
|
21
25
|
|
|
22
26
|
from __future__ import annotations
|
|
23
27
|
|
|
28
|
+
import io
|
|
24
29
|
import json
|
|
25
30
|
import logging
|
|
31
|
+
import math
|
|
26
32
|
import os
|
|
33
|
+
import re
|
|
27
34
|
import time
|
|
28
35
|
from dataclasses import dataclass
|
|
36
|
+
from datetime import datetime
|
|
29
37
|
from pathlib import Path
|
|
30
|
-
from typing import IO, Dict, List, Optional, Sequence
|
|
38
|
+
from typing import IO, Dict, List, Optional, Sequence, Set, Tuple
|
|
31
39
|
|
|
32
40
|
import pandas as pd
|
|
33
41
|
|
|
34
42
|
from ipmg.core.engine import HostResult
|
|
43
|
+
from ipmg.exceptions import FileIOError
|
|
35
44
|
from ipmg.reporting.frames import RESULT_COLUMNS, format_open_ports
|
|
36
|
-
from ipmg.utils.helpers import spreadsheet_escape
|
|
45
|
+
from ipmg.utils.helpers import FORMULA_PREFIXES, spreadsheet_escape
|
|
37
46
|
|
|
38
47
|
log = logging.getLogger(__name__)
|
|
39
48
|
|
|
@@ -48,6 +57,12 @@ REPORT_FORMATS = ("xlsx", "csv", "json", "jsonl", "md")
|
|
|
48
57
|
STREAMING_FORMATS = ("csv", "jsonl")
|
|
49
58
|
#: Formats that only exist as a whole file, so they are re-snapshotted.
|
|
50
59
|
SNAPSHOT_FORMATS = ("xlsx", "json", "md")
|
|
60
|
+
#: Formats a scan can be resumed from, most faithful first. ``md`` only
|
|
61
|
+
#: previews the first 25 hosts, so it cannot stand in for the whole report.
|
|
62
|
+
RESUMABLE_FORMATS = ("jsonl", "csv", "json", "xlsx")
|
|
63
|
+
|
|
64
|
+
#: ``<base>_<YYYYMMDD_HHMMSS>.<format>``, the name every report is saved under.
|
|
65
|
+
_REPORT_NAME = re.compile(r"^(?P<prefix>.+)_(?P<timestamp>\d{8}_\d{6})\.(?P<fmt>[a-z]+)$")
|
|
51
66
|
|
|
52
67
|
|
|
53
68
|
@dataclass(frozen=True)
|
|
@@ -120,6 +135,7 @@ class IncrementalReport:
|
|
|
120
135
|
batch_timestamp: object,
|
|
121
136
|
options: IncrementalOptions = IncrementalOptions(),
|
|
122
137
|
previous: Optional[Sequence[HostResult]] = None,
|
|
138
|
+
previous_elapsed_s: float = 0.0,
|
|
123
139
|
) -> None:
|
|
124
140
|
self.options = options.clamped()
|
|
125
141
|
self._base = base
|
|
@@ -127,13 +143,17 @@ class IncrementalReport:
|
|
|
127
143
|
self._batch_timestamp = batch_timestamp
|
|
128
144
|
self._formats = [fmt for fmt in dict.fromkeys(formats)]
|
|
129
145
|
self._results: List[HostResult] = list(previous or ())
|
|
146
|
+
# Resumed rows keep the time the earlier run had reached, and new rows
|
|
147
|
+
# count on from it, so a report interrupted twice still knows how long
|
|
148
|
+
# the whole scan has taken.
|
|
130
149
|
self._rows: List[Dict[str, object]] = [
|
|
131
|
-
result_row(result, batch_timestamp,
|
|
150
|
+
result_row(result, batch_timestamp, previous_elapsed_s if previous else None)
|
|
151
|
+
for result in self._results
|
|
132
152
|
]
|
|
133
153
|
self._handles: Dict[str, IO[str]] = {}
|
|
134
154
|
self._written: Dict[str, None] = {}
|
|
135
|
-
self._started_at = time.monotonic()
|
|
136
|
-
self._last_snapshot =
|
|
155
|
+
self._started_at = time.monotonic() - previous_elapsed_s
|
|
156
|
+
self._last_snapshot = time.monotonic()
|
|
137
157
|
self._closed = False
|
|
138
158
|
|
|
139
159
|
# -- lifecycle ---------------------------------------------------------
|
|
@@ -259,6 +279,194 @@ class IncrementalReport:
|
|
|
259
279
|
self._handles.clear()
|
|
260
280
|
|
|
261
281
|
|
|
282
|
+
@dataclass(frozen=True)
|
|
283
|
+
class PartialReport:
|
|
284
|
+
"""The hosts an interrupted scan left in its report, ready to resume."""
|
|
285
|
+
|
|
286
|
+
path: str
|
|
287
|
+
#: The ``--output`` prefix and timestamp the report was saved under, so a
|
|
288
|
+
#: resumed scan writes back to the same files.
|
|
289
|
+
base: str
|
|
290
|
+
timestamp: str
|
|
291
|
+
results: Tuple[HostResult, ...]
|
|
292
|
+
batch_timestamp: Optional[datetime]
|
|
293
|
+
#: How long the earlier run had been scanning when it stopped.
|
|
294
|
+
elapsed_s: float
|
|
295
|
+
|
|
296
|
+
@property
|
|
297
|
+
def scanned(self) -> Set[str]:
|
|
298
|
+
return {result.ip for result in self.results}
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def find_partial_report(base: str) -> Optional[str]:
|
|
302
|
+
"""The newest report saved under ``base`` that a scan can resume from.
|
|
303
|
+
|
|
304
|
+
When one scan left several formats behind, the most faithful one wins:
|
|
305
|
+
``jsonl`` and ``csv`` hold every host up to the moment of interruption,
|
|
306
|
+
while ``json`` and ``xlsx`` are only as recent as their last autosave.
|
|
307
|
+
"""
|
|
308
|
+
directory = Path(base).parent
|
|
309
|
+
prefix = Path(base).name
|
|
310
|
+
candidates = []
|
|
311
|
+
try:
|
|
312
|
+
entries = list(directory.iterdir())
|
|
313
|
+
except OSError:
|
|
314
|
+
return None
|
|
315
|
+
for entry in entries:
|
|
316
|
+
match = _REPORT_NAME.match(entry.name)
|
|
317
|
+
if not match or match["prefix"] != prefix or match["fmt"] not in RESUMABLE_FORMATS:
|
|
318
|
+
continue
|
|
319
|
+
rank = RESUMABLE_FORMATS.index(match["fmt"])
|
|
320
|
+
candidates.append((match["timestamp"], -rank, str(entry)))
|
|
321
|
+
return max(candidates)[2] if candidates else None
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def load_partial_report(path: str) -> PartialReport:
|
|
325
|
+
"""Read the hosts a report already holds, including one cut off mid-write."""
|
|
326
|
+
name = _REPORT_NAME.match(Path(path).name)
|
|
327
|
+
if name is None:
|
|
328
|
+
raise FileIOError(
|
|
329
|
+
f"Cannot resume from {path}: expected a report named like "
|
|
330
|
+
"results_20260917_120000.jsonl, as IPMG saves them."
|
|
331
|
+
)
|
|
332
|
+
fmt = name["fmt"]
|
|
333
|
+
if fmt not in RESUMABLE_FORMATS:
|
|
334
|
+
raise FileIOError(
|
|
335
|
+
f"Cannot resume from a .{fmt} report; use the "
|
|
336
|
+
f"{', '.join(RESUMABLE_FORMATS)} report from the same scan instead."
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
try:
|
|
340
|
+
frame = _read_report(path, fmt)
|
|
341
|
+
except FileNotFoundError:
|
|
342
|
+
raise FileIOError(f"Report to resume not found: {path}") from None
|
|
343
|
+
except (OSError, ValueError) as exc:
|
|
344
|
+
raise FileIOError(f"Cannot read report {path}: {exc}") from exc
|
|
345
|
+
|
|
346
|
+
missing = [column for column in ("IP Address", "Status") if column not in frame.columns]
|
|
347
|
+
if missing:
|
|
348
|
+
raise FileIOError(f"{path} is not an IPMG report: missing {', '.join(missing)}.")
|
|
349
|
+
|
|
350
|
+
# A spreadsheet format stores cells escaped against formula injection;
|
|
351
|
+
# the scan itself needs the text the host actually reported.
|
|
352
|
+
unescape = fmt in ("csv", "xlsx")
|
|
353
|
+
by_ip: Dict[str, HostResult] = {}
|
|
354
|
+
for row in frame.to_dict(orient="records"):
|
|
355
|
+
result = _row_result(row, unescape)
|
|
356
|
+
if result is not None:
|
|
357
|
+
by_ip[result.ip] = result
|
|
358
|
+
|
|
359
|
+
base = str(Path(path).parent / name["prefix"])
|
|
360
|
+
return PartialReport(
|
|
361
|
+
path=path,
|
|
362
|
+
base=base,
|
|
363
|
+
timestamp=name["timestamp"],
|
|
364
|
+
results=tuple(by_ip.values()),
|
|
365
|
+
batch_timestamp=_batch_timestamp(frame),
|
|
366
|
+
elapsed_s=_elapsed(frame),
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _read_report(path: str, fmt: str) -> pd.DataFrame:
|
|
371
|
+
if fmt == "xlsx":
|
|
372
|
+
return pd.read_excel(path, dtype=object)
|
|
373
|
+
if fmt == "json":
|
|
374
|
+
with open(path, encoding="utf-8") as handle:
|
|
375
|
+
return pd.DataFrame(json.load(handle))
|
|
376
|
+
|
|
377
|
+
with open(path, encoding="utf-8", newline="") as handle:
|
|
378
|
+
text = handle.read()
|
|
379
|
+
# Rows are flushed whole, so a last line without its newline is one the
|
|
380
|
+
# previous run was killed while writing. It is dropped, and that host is
|
|
381
|
+
# simply scanned again.
|
|
382
|
+
if text and not text.endswith("\n"):
|
|
383
|
+
text = text[: text.rfind("\n") + 1]
|
|
384
|
+
|
|
385
|
+
if fmt == "csv":
|
|
386
|
+
if not text.strip():
|
|
387
|
+
return pd.DataFrame(columns=RESULT_COLUMNS)
|
|
388
|
+
return pd.read_csv(io.StringIO(text), dtype=str, keep_default_na=False)
|
|
389
|
+
|
|
390
|
+
records = []
|
|
391
|
+
for number, line in enumerate(text.splitlines(), start=1):
|
|
392
|
+
if not line.strip():
|
|
393
|
+
continue
|
|
394
|
+
try:
|
|
395
|
+
records.append(json.loads(line))
|
|
396
|
+
except json.JSONDecodeError as exc:
|
|
397
|
+
raise ValueError(f"line {number} is not valid JSON ({exc.msg})") from None
|
|
398
|
+
return pd.DataFrame(records)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _is_blank(value: object) -> bool:
|
|
402
|
+
return value is None or (isinstance(value, float) and math.isnan(value)) or value == ""
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _text(value: object, unescape: bool = False) -> str:
|
|
406
|
+
if _is_blank(value):
|
|
407
|
+
return ""
|
|
408
|
+
text = str(value).strip()
|
|
409
|
+
if unescape and text.startswith("'") and text[1:].startswith(FORMULA_PREFIXES):
|
|
410
|
+
return text[1:]
|
|
411
|
+
return text
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _latency(value: object) -> Optional[float]:
|
|
415
|
+
if _is_blank(value):
|
|
416
|
+
return None
|
|
417
|
+
try:
|
|
418
|
+
return float(value) # type: ignore[arg-type]
|
|
419
|
+
except (TypeError, ValueError):
|
|
420
|
+
return None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _open_ports(value: object) -> Tuple[int, ...]:
|
|
424
|
+
ports = []
|
|
425
|
+
for piece in _text(value).split(","):
|
|
426
|
+
try:
|
|
427
|
+
ports.append(int(float(piece)))
|
|
428
|
+
except ValueError:
|
|
429
|
+
continue
|
|
430
|
+
return tuple(ports)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _row_result(row: Dict[str, object], unescape: bool) -> Optional[HostResult]:
|
|
434
|
+
ip = _text(row.get("IP Address"))
|
|
435
|
+
status = _text(row.get("Status"))
|
|
436
|
+
if not ip or not status:
|
|
437
|
+
return None
|
|
438
|
+
return HostResult(
|
|
439
|
+
ip=ip,
|
|
440
|
+
status=status,
|
|
441
|
+
latency=_latency(row.get("Latency")),
|
|
442
|
+
hostname=_text(row.get("Hostname"), unescape),
|
|
443
|
+
open_ports=_open_ports(row.get("Open Ports")),
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _batch_timestamp(frame: pd.DataFrame) -> Optional[datetime]:
|
|
448
|
+
"""The resumed scan's start time, so both runs report as one batch."""
|
|
449
|
+
if "Batch Timestamp" not in frame.columns:
|
|
450
|
+
return None
|
|
451
|
+
for value in frame["Batch Timestamp"]:
|
|
452
|
+
if _is_blank(value):
|
|
453
|
+
continue
|
|
454
|
+
try:
|
|
455
|
+
# A finished json report stores it as epoch milliseconds.
|
|
456
|
+
unit = "ms" if isinstance(value, (int, float)) else None
|
|
457
|
+
return pd.to_datetime(value, unit=unit).to_pydatetime()
|
|
458
|
+
except (TypeError, ValueError, OverflowError):
|
|
459
|
+
return None
|
|
460
|
+
return None
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _elapsed(frame: pd.DataFrame) -> float:
|
|
464
|
+
if "Scan Duration (s)" not in frame.columns:
|
|
465
|
+
return 0.0
|
|
466
|
+
durations = pd.to_numeric(frame["Scan Duration (s)"], errors="coerce").dropna()
|
|
467
|
+
return float(durations.max()) if not durations.empty else 0.0
|
|
468
|
+
|
|
469
|
+
|
|
262
470
|
def atomic_write_bytes(path: str, data: bytes) -> None:
|
|
263
471
|
"""Replace ``path`` with ``data`` in one step, or leave it untouched.
|
|
264
472
|
|
|
@@ -111,7 +111,8 @@ def heading(title: str) -> None:
|
|
|
111
111
|
def field(label: str, value: object, value_style: str = "ipmg.value") -> None:
|
|
112
112
|
"""One aligned ``label value`` line."""
|
|
113
113
|
text = Text(INDENT)
|
|
114
|
-
|
|
114
|
+
# A label as long as the column still gets one space before its value.
|
|
115
|
+
text.append(f"{label:<{LABEL_WIDTH - 1}} ", style="ipmg.label")
|
|
115
116
|
text.append(str(value), style=value_style)
|
|
116
117
|
console.print(text)
|
|
117
118
|
|