httpcheck 1.4.3__tar.gz → 1.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpcheck-1.4.3/httpcheck.egg-info → httpcheck-1.5.1}/PKG-INFO +83 -16
- httpcheck-1.4.3/PKG-INFO → httpcheck-1.5.1/README.md +64 -55
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/__init__.py +4 -1
- httpcheck-1.5.1/httpcheck/async_site_checker.py +238 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/cli.py +103 -45
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/common.py +4 -6
- httpcheck-1.5.1/httpcheck/config.py +143 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/file_handler.py +6 -7
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/logger.py +2 -3
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/output_formatter.py +0 -1
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/site_checker.py +70 -38
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/tld_manager.py +48 -25
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/validation.py +12 -12
- httpcheck-1.4.3/README.md → httpcheck-1.5.1/httpcheck.egg-info/PKG-INFO +123 -11
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/SOURCES.txt +4 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/requires.txt +16 -2
- {httpcheck-1.4.3 → httpcheck-1.5.1}/pyproject.toml +27 -6
- httpcheck-1.5.1/tests/test_async_site_checker.py +360 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_cli_integration.py +250 -28
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_common.py +2 -2
- httpcheck-1.5.1/tests/test_config.py +323 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_httpcheck.py +1 -1
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_init.py +2 -2
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_performance.py +22 -4
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_site_checker.py +151 -8
- httpcheck-1.5.1/tests/test_tld_manager.py +357 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_validation.py +5 -1
- httpcheck-1.4.3/tests/test_tld_manager.py +0 -101
- {httpcheck-1.4.3 → httpcheck-1.5.1}/LICENSE +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/notification.py +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/dependency_links.txt +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/entry_points.txt +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/top_level.txt +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/setup.cfg +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_file_handler.py +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_output_formats.py +0 -0
- {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_request_customization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: httpcheck
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.1
|
|
4
4
|
Summary: HTTP status code checker with advanced features
|
|
5
5
|
Author-email: Thomas Juul Dyhr <thomas@dyhr.com>
|
|
6
6
|
License: MIT
|
|
@@ -13,19 +13,27 @@ Classifier: Intended Audience :: System Administrators
|
|
|
13
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
21
|
Classifier: Topic :: System :: Monitoring
|
|
21
22
|
Classifier: Topic :: System :: Networking :: Monitoring
|
|
22
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.10
|
|
23
24
|
Description-Content-Type: text/markdown
|
|
24
25
|
License-File: LICENSE
|
|
25
|
-
Requires-Dist: requests>=2.
|
|
26
|
+
Requires-Dist: requests>=2.33.0
|
|
27
|
+
Requires-Dist: httpx>=0.27.0
|
|
26
28
|
Requires-Dist: tabulate>=0.9.0
|
|
27
29
|
Requires-Dist: tqdm>=4.67.0
|
|
28
30
|
Requires-Dist: validators>=0.22.0
|
|
31
|
+
Requires-Dist: tomli>=2.0.0; python_version < "3.11"
|
|
32
|
+
Requires-Dist: urllib3>=2.8.0
|
|
33
|
+
Requires-Dist: certifi>=2026.10.19
|
|
34
|
+
Requires-Dist: idna>=3.15
|
|
35
|
+
Requires-Dist: anyio>=4.14.2
|
|
36
|
+
Requires-Dist: h11>=0.16.0
|
|
29
37
|
Provides-Extra: macos
|
|
30
38
|
Requires-Dist: rumps>=0.4.0; extra == "macos"
|
|
31
39
|
Requires-Dist: pyobjc-framework-Cocoa>=10.0; extra == "macos"
|
|
@@ -37,12 +45,27 @@ Requires-Dist: pytest>=7.0; extra == "dev"
|
|
|
37
45
|
Requires-Dist: pytest-mock>=3.0; extra == "dev"
|
|
38
46
|
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
39
47
|
Requires-Dist: pytest-benchmark>=5.0; extra == "dev"
|
|
48
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
|
|
40
49
|
Requires-Dist: pylint>=3.1.0; extra == "dev"
|
|
41
|
-
Requires-Dist: mypy
|
|
50
|
+
Requires-Dist: mypy==2.4.0; extra == "dev"
|
|
51
|
+
Requires-Dist: types-requests>=2.31.0; extra == "dev"
|
|
52
|
+
Requires-Dist: types-tabulate>=0.9.0; extra == "dev"
|
|
53
|
+
Requires-Dist: types-tqdm>=4.66.0; extra == "dev"
|
|
54
|
+
Requires-Dist: tomli>=2.0.0; extra == "dev"
|
|
42
55
|
Provides-Extra: build
|
|
43
56
|
Requires-Dist: py2app>=0.28; extra == "build"
|
|
44
57
|
Dynamic: license-file
|
|
45
58
|
|
|
59
|
+
<div align="right">
|
|
60
|
+
|
|
61
|
+
<a href="https://railway.com?referralCode=QhjuBc">
|
|
62
|
+
|
|
63
|
+
<img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
|
|
64
|
+
|
|
65
|
+
</a>
|
|
66
|
+
|
|
67
|
+
</div>
|
|
68
|
+
|
|
46
69
|
[](https://badge.fury.io/py/httpcheck)
|
|
47
70
|
[](https://pypi.org/project/httpcheck/)
|
|
48
71
|
[](https://pypi.org/project/httpcheck/)
|
|
@@ -51,15 +74,16 @@ Dynamic: license-file
|
|
|
51
74
|
[](https://github.com/docdyhr/httpcheck/issues)
|
|
52
75
|

|
|
53
76
|

|
|
77
|
+
[](https://httpcheck.readthedocs.io/)
|
|
54
78
|
|
|
55
79
|
# Check Websites HTTP Status Codes with httpcheck
|
|
56
80
|
|
|
57
81
|
<img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
|
|
58
82
|
|
|
59
83
|
* **Name**: httpcheck (CLI) / onSite (Menu Bar App)
|
|
60
|
-
* **Current Version**: 1.
|
|
61
|
-
* **
|
|
62
|
-
* **Programming Language**: Python 3.
|
|
84
|
+
* **Current Version**: 1.5.1 (security patch for 1.5.0: Async I/O & Configuration)
|
|
85
|
+
* **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
|
|
86
|
+
* **Programming Language**: Python 3.10+
|
|
63
87
|
* **Author**: Thomas Juul Dyhr
|
|
64
88
|
* **Purpose**: Advanced HTTP status checker with monitoring capabilities
|
|
65
89
|
* **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
|
|
@@ -67,9 +91,25 @@ Dynamic: license-file
|
|
|
67
91
|
|
|
68
92
|
## 🚀 Release Status
|
|
69
93
|
|
|
70
|
-
**httpcheck v1.
|
|
71
|
-
|
|
72
|
-
✅ **
|
|
94
|
+
**httpcheck v1.5.1 has been released! 🎉**
|
|
95
|
+
|
|
96
|
+
✅ **v1.5.1 (security patch):** installing httpcheck now also requires patched
|
|
97
|
+
versions of `urllib3`, `certifi`, `idna`, `anyio` and `h11`, so an install into
|
|
98
|
+
an existing environment cannot keep vulnerable copies. No behaviour changes.
|
|
99
|
+
|
|
100
|
+
✅ **NEW in v1.5.0:**
|
|
101
|
+
- **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
|
|
102
|
+
`./.httpcheck.toml` (the project file overrides the user file); CLI flags always
|
|
103
|
+
take precedence
|
|
104
|
+
- **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
|
|
105
|
+
the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
|
|
106
|
+
faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
|
|
107
|
+
- **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
|
|
108
|
+
installing 1.4.3); Python 3.13 and 3.14 are tested in CI
|
|
109
|
+
- **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
|
|
110
|
+
work (1.4.3 reported the site as failed without retrying)
|
|
111
|
+
|
|
112
|
+
✅ **v1.4.2-1.4.3 (Published to PyPI):**
|
|
73
113
|
- **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
|
|
74
114
|
- **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
|
|
75
115
|
- **Professional Documentation**: 24-page documentation with ReadTheDocs theme
|
|
@@ -83,15 +123,13 @@ Dynamic: license-file
|
|
|
83
123
|
- **Multiple Output Formats**: JSON and CSV support
|
|
84
124
|
- **Advanced Request Control**: Custom headers, SSL options
|
|
85
125
|
|
|
86
|
-
|
|
87
|
-
- **Async I/O**: 2-3x performance improvement for large site lists
|
|
88
|
-
- **Configuration Files**: User-defined defaults and settings
|
|
126
|
+
🔭 **Next: v1.6.0 (target Q1 2027)**
|
|
89
127
|
- **Monitoring Mode**: Continuous site monitoring with notifications
|
|
90
128
|
- **Enhanced UX**: Colored output and improved progress reporting
|
|
91
129
|
|
|
92
130
|
See detailed plans in:
|
|
93
|
-
- [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
|
|
94
|
-
- [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
|
|
131
|
+
- [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
|
|
132
|
+
- [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
|
|
95
133
|
- [TODO.md](TODO.md) - Current prioritized task list
|
|
96
134
|
- [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
|
|
97
135
|
|
|
@@ -399,6 +437,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
|
|
|
399
437
|
httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
|
|
400
438
|
```
|
|
401
439
|
|
|
440
|
+
### configuration file
|
|
441
|
+
|
|
442
|
+
httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
|
|
443
|
+
|
|
444
|
+
```toml
|
|
445
|
+
# ~/.httpcheck.toml
|
|
446
|
+
|
|
447
|
+
[defaults]
|
|
448
|
+
timeout = 10.0
|
|
449
|
+
retries = 3
|
|
450
|
+
follow_redirects = "always" # always | never | http-only | https-only
|
|
451
|
+
output_format = "table" # table | json | csv
|
|
452
|
+
verify_ssl = true
|
|
453
|
+
workers = 20
|
|
454
|
+
retry_delay = 0.5
|
|
455
|
+
max_redirects = 10
|
|
456
|
+
|
|
457
|
+
[headers]
|
|
458
|
+
User-Agent = "mybot/1.0"
|
|
459
|
+
Accept = "application/json"
|
|
460
|
+
|
|
461
|
+
[notifications]
|
|
462
|
+
enabled = true
|
|
463
|
+
on_failure = true
|
|
464
|
+
sound = "Ping"
|
|
465
|
+
```
|
|
466
|
+
|
|
467
|
+
Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
|
|
468
|
+
|
|
402
469
|
### file input examples
|
|
403
470
|
|
|
404
471
|
Create a domains file with comments and use it with httpcheck:
|
|
@@ -1,47 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Classifier: Development Status :: 4 - Beta
|
|
11
|
-
Classifier: Environment :: Console
|
|
12
|
-
Classifier: Intended Audience :: System Administrators
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
-
Classifier: Topic :: System :: Monitoring
|
|
21
|
-
Classifier: Topic :: System :: Networking :: Monitoring
|
|
22
|
-
Requires-Python: >=3.9
|
|
23
|
-
Description-Content-Type: text/markdown
|
|
24
|
-
License-File: LICENSE
|
|
25
|
-
Requires-Dist: requests>=2.32.5
|
|
26
|
-
Requires-Dist: tabulate>=0.9.0
|
|
27
|
-
Requires-Dist: tqdm>=4.67.0
|
|
28
|
-
Requires-Dist: validators>=0.22.0
|
|
29
|
-
Provides-Extra: macos
|
|
30
|
-
Requires-Dist: rumps>=0.4.0; extra == "macos"
|
|
31
|
-
Requires-Dist: pyobjc-framework-Cocoa>=10.0; extra == "macos"
|
|
32
|
-
Requires-Dist: pyobjc-framework-Foundation>=10.0; extra == "macos"
|
|
33
|
-
Requires-Dist: pyobjc-framework-AppKit>=10.0; extra == "macos"
|
|
34
|
-
Requires-Dist: pyobjc-core>=10.0; extra == "macos"
|
|
35
|
-
Provides-Extra: dev
|
|
36
|
-
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
37
|
-
Requires-Dist: pytest-mock>=3.0; extra == "dev"
|
|
38
|
-
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
39
|
-
Requires-Dist: pytest-benchmark>=5.0; extra == "dev"
|
|
40
|
-
Requires-Dist: pylint>=3.1.0; extra == "dev"
|
|
41
|
-
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
42
|
-
Provides-Extra: build
|
|
43
|
-
Requires-Dist: py2app>=0.28; extra == "build"
|
|
44
|
-
Dynamic: license-file
|
|
1
|
+
<div align="right">
|
|
2
|
+
|
|
3
|
+
<a href="https://railway.com?referralCode=QhjuBc">
|
|
4
|
+
|
|
5
|
+
<img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
|
|
6
|
+
|
|
7
|
+
</a>
|
|
8
|
+
|
|
9
|
+
</div>
|
|
45
10
|
|
|
46
11
|
[](https://badge.fury.io/py/httpcheck)
|
|
47
12
|
[](https://pypi.org/project/httpcheck/)
|
|
@@ -51,15 +16,16 @@ Dynamic: license-file
|
|
|
51
16
|
[](https://github.com/docdyhr/httpcheck/issues)
|
|
52
17
|

|
|
53
18
|

|
|
19
|
+
[](https://httpcheck.readthedocs.io/)
|
|
54
20
|
|
|
55
21
|
# Check Websites HTTP Status Codes with httpcheck
|
|
56
22
|
|
|
57
23
|
<img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
|
|
58
24
|
|
|
59
25
|
* **Name**: httpcheck (CLI) / onSite (Menu Bar App)
|
|
60
|
-
* **Current Version**: 1.
|
|
61
|
-
* **
|
|
62
|
-
* **Programming Language**: Python 3.
|
|
26
|
+
* **Current Version**: 1.5.1 (security patch for 1.5.0: Async I/O & Configuration)
|
|
27
|
+
* **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
|
|
28
|
+
* **Programming Language**: Python 3.10+
|
|
63
29
|
* **Author**: Thomas Juul Dyhr
|
|
64
30
|
* **Purpose**: Advanced HTTP status checker with monitoring capabilities
|
|
65
31
|
* **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
|
|
@@ -67,9 +33,25 @@ Dynamic: license-file
|
|
|
67
33
|
|
|
68
34
|
## 🚀 Release Status
|
|
69
35
|
|
|
70
|
-
**httpcheck v1.
|
|
71
|
-
|
|
72
|
-
✅ **
|
|
36
|
+
**httpcheck v1.5.1 has been released! 🎉**
|
|
37
|
+
|
|
38
|
+
✅ **v1.5.1 (security patch):** installing httpcheck now also requires patched
|
|
39
|
+
versions of `urllib3`, `certifi`, `idna`, `anyio` and `h11`, so an install into
|
|
40
|
+
an existing environment cannot keep vulnerable copies. No behaviour changes.
|
|
41
|
+
|
|
42
|
+
✅ **NEW in v1.5.0:**
|
|
43
|
+
- **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
|
|
44
|
+
`./.httpcheck.toml` (the project file overrides the user file); CLI flags always
|
|
45
|
+
take precedence
|
|
46
|
+
- **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
|
|
47
|
+
the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
|
|
48
|
+
faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
|
|
49
|
+
- **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
|
|
50
|
+
installing 1.4.3); Python 3.13 and 3.14 are tested in CI
|
|
51
|
+
- **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
|
|
52
|
+
work (1.4.3 reported the site as failed without retrying)
|
|
53
|
+
|
|
54
|
+
✅ **v1.4.2-1.4.3 (Published to PyPI):**
|
|
73
55
|
- **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
|
|
74
56
|
- **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
|
|
75
57
|
- **Professional Documentation**: 24-page documentation with ReadTheDocs theme
|
|
@@ -83,15 +65,13 @@ Dynamic: license-file
|
|
|
83
65
|
- **Multiple Output Formats**: JSON and CSV support
|
|
84
66
|
- **Advanced Request Control**: Custom headers, SSL options
|
|
85
67
|
|
|
86
|
-
|
|
87
|
-
- **Async I/O**: 2-3x performance improvement for large site lists
|
|
88
|
-
- **Configuration Files**: User-defined defaults and settings
|
|
68
|
+
🔭 **Next: v1.6.0 (target Q1 2027)**
|
|
89
69
|
- **Monitoring Mode**: Continuous site monitoring with notifications
|
|
90
70
|
- **Enhanced UX**: Colored output and improved progress reporting
|
|
91
71
|
|
|
92
72
|
See detailed plans in:
|
|
93
|
-
- [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
|
|
94
|
-
- [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
|
|
73
|
+
- [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
|
|
74
|
+
- [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
|
|
95
75
|
- [TODO.md](TODO.md) - Current prioritized task list
|
|
96
76
|
- [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
|
|
97
77
|
|
|
@@ -399,6 +379,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
|
|
|
399
379
|
httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
|
|
400
380
|
```
|
|
401
381
|
|
|
382
|
+
### configuration file
|
|
383
|
+
|
|
384
|
+
httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
|
|
385
|
+
|
|
386
|
+
```toml
|
|
387
|
+
# ~/.httpcheck.toml
|
|
388
|
+
|
|
389
|
+
[defaults]
|
|
390
|
+
timeout = 10.0
|
|
391
|
+
retries = 3
|
|
392
|
+
follow_redirects = "always" # always | never | http-only | https-only
|
|
393
|
+
output_format = "table" # table | json | csv
|
|
394
|
+
verify_ssl = true
|
|
395
|
+
workers = 20
|
|
396
|
+
retry_delay = 0.5
|
|
397
|
+
max_redirects = 10
|
|
398
|
+
|
|
399
|
+
[headers]
|
|
400
|
+
User-Agent = "mybot/1.0"
|
|
401
|
+
Accept = "application/json"
|
|
402
|
+
|
|
403
|
+
[notifications]
|
|
404
|
+
enabled = true
|
|
405
|
+
on_failure = true
|
|
406
|
+
sound = "Ping"
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
|
|
410
|
+
|
|
402
411
|
### file input examples
|
|
403
412
|
|
|
404
413
|
Create a domains file with comments and use it with httpcheck:
|
|
@@ -5,13 +5,14 @@ A Python CLI tool for checking HTTP status codes of websites with
|
|
|
5
5
|
threading, redirect handling, TLD validation, and system notifications.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
__version__ = "1.
|
|
8
|
+
__version__ = "1.5.1"
|
|
9
9
|
__author__ = "Thomas Juul Dyhr"
|
|
10
10
|
__email__ = "thomas@dyhr.com"
|
|
11
11
|
|
|
12
12
|
# Import specific items for public API
|
|
13
13
|
from .cli import main
|
|
14
14
|
from .common import VERSION, InvalidTLDException, SiteStatus
|
|
15
|
+
from .config import DEFAULT_CONFIG, load_config
|
|
15
16
|
from .file_handler import FileInputHandler, url_validation
|
|
16
17
|
from .notification import notify
|
|
17
18
|
from .output_formatter import (
|
|
@@ -37,6 +38,8 @@ from .validation import (
|
|
|
37
38
|
__all__ = [
|
|
38
39
|
# Core functionality
|
|
39
40
|
"check_site",
|
|
41
|
+
"load_config",
|
|
42
|
+
"DEFAULT_CONFIG",
|
|
40
43
|
"print_format",
|
|
41
44
|
"format_json",
|
|
42
45
|
"format_csv",
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Async HTTP site checking for httpcheck."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from urllib.parse import urlparse
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from .common import STATUS_CODES, VERSION, SiteStatus
|
|
13
|
+
|
|
14
|
+
DEFAULT_LIMITS = httpx.Limits(max_connections=50, max_keepalive_connections=20)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _create_headers(custom_headers: dict | None = None) -> dict:
|
|
18
|
+
"""Create default headers including User-Agent."""
|
|
19
|
+
headers = {"User-Agent": f"httpcheck Async Agent {VERSION}"}
|
|
20
|
+
if custom_headers:
|
|
21
|
+
headers.update(custom_headers)
|
|
22
|
+
return headers
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _extract_domain(url_like: str | httpx.URL, fallback: str) -> str:
|
|
26
|
+
"""Return host component from httpx.URL or string URL."""
|
|
27
|
+
try:
|
|
28
|
+
if isinstance(url_like, httpx.URL):
|
|
29
|
+
return str(url_like.host)
|
|
30
|
+
parsed = urlparse(str(url_like))
|
|
31
|
+
return parsed.hostname or fallback
|
|
32
|
+
except (AttributeError, ValueError):
|
|
33
|
+
return fallback
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _extract_redirects(
|
|
37
|
+
response: httpx.Response,
|
|
38
|
+
) -> tuple[list[tuple[str, int]], list[tuple[str, int, float]]]:
|
|
39
|
+
"""Return redirect chain and timing from an httpx response."""
|
|
40
|
+
chain: list[tuple[str, int]] = []
|
|
41
|
+
timing: list[tuple[str, int, float]] = []
|
|
42
|
+
for hop in response.history:
|
|
43
|
+
chain.append((str(hop.url), hop.status_code))
|
|
44
|
+
timing.append((str(hop.url), hop.status_code, 0.0))
|
|
45
|
+
chain.append((str(response.url), response.status_code))
|
|
46
|
+
timing.append((str(response.url), response.status_code, 0.0))
|
|
47
|
+
return chain, timing
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _should_stop_redirect(follow_redirects: str, redirect_url: httpx.URL) -> bool:
|
|
51
|
+
"""Return True when protocol policy blocks following this redirect."""
|
|
52
|
+
if follow_redirects == "http-only" and redirect_url.scheme == "https":
|
|
53
|
+
return True
|
|
54
|
+
if follow_redirects == "https-only" and redirect_url.scheme == "http":
|
|
55
|
+
return True
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def _manual_redirect_flow(
|
|
60
|
+
client: httpx.AsyncClient,
|
|
61
|
+
url: str,
|
|
62
|
+
*,
|
|
63
|
+
timeout: float,
|
|
64
|
+
max_redirects: int,
|
|
65
|
+
follow_redirects: str,
|
|
66
|
+
) -> tuple[httpx.Response, float, list[tuple[str, int]], list[tuple[str, int, float]]]:
|
|
67
|
+
"""Follow redirects manually to enforce protocol restrictions."""
|
|
68
|
+
chain: list[tuple[str, int]] = []
|
|
69
|
+
timing: list[tuple[str, int, float]] = []
|
|
70
|
+
|
|
71
|
+
current_url: httpx.URL = httpx.URL(url)
|
|
72
|
+
total_start = time.monotonic()
|
|
73
|
+
redirects = 0
|
|
74
|
+
while True:
|
|
75
|
+
hop_start = time.monotonic()
|
|
76
|
+
response = await client.get(
|
|
77
|
+
str(current_url),
|
|
78
|
+
follow_redirects=False,
|
|
79
|
+
timeout=timeout,
|
|
80
|
+
)
|
|
81
|
+
hop_elapsed = time.monotonic() - hop_start
|
|
82
|
+
|
|
83
|
+
chain.append((str(response.url), response.status_code))
|
|
84
|
+
timing.append((str(response.url), response.status_code, hop_elapsed))
|
|
85
|
+
|
|
86
|
+
if not response.is_redirect or "location" not in response.headers:
|
|
87
|
+
break
|
|
88
|
+
|
|
89
|
+
if redirects >= max_redirects:
|
|
90
|
+
break
|
|
91
|
+
|
|
92
|
+
location = response.headers.get("location", "")
|
|
93
|
+
base_url = (
|
|
94
|
+
response.url
|
|
95
|
+
if isinstance(response.url, httpx.URL)
|
|
96
|
+
else httpx.URL(str(response.url))
|
|
97
|
+
)
|
|
98
|
+
next_url = base_url.join(location)
|
|
99
|
+
if _should_stop_redirect(follow_redirects, next_url):
|
|
100
|
+
break
|
|
101
|
+
|
|
102
|
+
redirects += 1
|
|
103
|
+
current_url = next_url
|
|
104
|
+
|
|
105
|
+
total_elapsed = time.monotonic() - total_start
|
|
106
|
+
return response, total_elapsed, chain, timing
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def async_check_site(
|
|
110
|
+
site: str,
|
|
111
|
+
timeout: float = 5.0,
|
|
112
|
+
retries: int = 2,
|
|
113
|
+
*,
|
|
114
|
+
follow_redirects: str = "always",
|
|
115
|
+
max_redirects: int = 30,
|
|
116
|
+
custom_headers: dict | None = None,
|
|
117
|
+
verify_ssl: bool = True,
|
|
118
|
+
retry_delay: float = 1.0,
|
|
119
|
+
client: httpx.AsyncClient | None = None,
|
|
120
|
+
) -> SiteStatus:
|
|
121
|
+
"""Async version of check_site with optional protocol-restricted redirects."""
|
|
122
|
+
headers = _create_headers(custom_headers)
|
|
123
|
+
allow_redirects = follow_redirects == "always"
|
|
124
|
+
manual_redirects = follow_redirects in {"http-only", "https-only"}
|
|
125
|
+
|
|
126
|
+
async def _execute_request(
|
|
127
|
+
active_client: httpx.AsyncClient,
|
|
128
|
+
) -> tuple[httpx.Response, float]:
|
|
129
|
+
start = time.monotonic()
|
|
130
|
+
response = await active_client.get(
|
|
131
|
+
site,
|
|
132
|
+
headers=headers,
|
|
133
|
+
timeout=timeout,
|
|
134
|
+
follow_redirects=allow_redirects,
|
|
135
|
+
)
|
|
136
|
+
elapsed = time.monotonic() - start
|
|
137
|
+
return response, elapsed
|
|
138
|
+
|
|
139
|
+
# httpx sets max_redirects per client, not per request: a caller-supplied
|
|
140
|
+
# client keeps its own limit.
|
|
141
|
+
owns_client = client is None
|
|
142
|
+
active_client = client or httpx.AsyncClient(
|
|
143
|
+
verify=verify_ssl,
|
|
144
|
+
limits=DEFAULT_LIMITS,
|
|
145
|
+
headers=headers,
|
|
146
|
+
max_redirects=max_redirects,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
for attempt in range(retries + 1):
|
|
151
|
+
try:
|
|
152
|
+
if manual_redirects:
|
|
153
|
+
(
|
|
154
|
+
response,
|
|
155
|
+
response_time,
|
|
156
|
+
redirect_chain,
|
|
157
|
+
redirect_timing,
|
|
158
|
+
) = await _manual_redirect_flow(
|
|
159
|
+
active_client,
|
|
160
|
+
site,
|
|
161
|
+
timeout=timeout,
|
|
162
|
+
max_redirects=max_redirects,
|
|
163
|
+
follow_redirects=follow_redirects,
|
|
164
|
+
)
|
|
165
|
+
else:
|
|
166
|
+
response, response_time = await _execute_request(active_client)
|
|
167
|
+
redirect_chain, redirect_timing = _extract_redirects(response)
|
|
168
|
+
|
|
169
|
+
domain = _extract_domain(getattr(response, "url", site), site)
|
|
170
|
+
return SiteStatus(
|
|
171
|
+
domain=domain,
|
|
172
|
+
status=str(response.status_code),
|
|
173
|
+
message=STATUS_CODES.get(str(response.status_code), "Unknown"),
|
|
174
|
+
redirect_chain=redirect_chain,
|
|
175
|
+
response_time=response_time,
|
|
176
|
+
redirect_timing=redirect_timing,
|
|
177
|
+
)
|
|
178
|
+
except (httpx.RequestError, httpx.HTTPError, ValueError, OSError) as exc:
|
|
179
|
+
if attempt == retries:
|
|
180
|
+
status = (
|
|
181
|
+
"[timeout]"
|
|
182
|
+
if isinstance(exc, httpx.TimeoutException)
|
|
183
|
+
else "[connection error]"
|
|
184
|
+
)
|
|
185
|
+
domain = site
|
|
186
|
+
if isinstance(exc, httpx.HTTPError):
|
|
187
|
+
try:
|
|
188
|
+
# .request raises RuntimeError when it was never set.
|
|
189
|
+
domain = _extract_domain(exc.request.url, site)
|
|
190
|
+
except RuntimeError:
|
|
191
|
+
pass
|
|
192
|
+
return SiteStatus(domain, status, str(exc))
|
|
193
|
+
if retry_delay > 0:
|
|
194
|
+
await asyncio.sleep(retry_delay)
|
|
195
|
+
# Only reached when retries < 0, as in the sync check_site.
|
|
196
|
+
return SiteStatus(site, "[unknown error]", "All retries failed")
|
|
197
|
+
finally:
|
|
198
|
+
if owns_client:
|
|
199
|
+
await active_client.aclose()
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
async def async_check_sites(
|
|
203
|
+
sites: Iterable[str],
|
|
204
|
+
*,
|
|
205
|
+
timeout: float = 5.0,
|
|
206
|
+
retries: int = 2,
|
|
207
|
+
follow_redirects: str = "always",
|
|
208
|
+
max_redirects: int = 30,
|
|
209
|
+
custom_headers: dict | None = None,
|
|
210
|
+
verify_ssl: bool = True,
|
|
211
|
+
retry_delay: float = 1.0,
|
|
212
|
+
concurrency: int = 50,
|
|
213
|
+
) -> list[SiteStatus]:
|
|
214
|
+
"""Check many sites concurrently with a shared client and semaphore."""
|
|
215
|
+
sem = asyncio.Semaphore(concurrency)
|
|
216
|
+
headers = _create_headers(custom_headers)
|
|
217
|
+
async with httpx.AsyncClient(
|
|
218
|
+
verify=verify_ssl,
|
|
219
|
+
limits=DEFAULT_LIMITS,
|
|
220
|
+
headers=headers,
|
|
221
|
+
max_redirects=max_redirects,
|
|
222
|
+
) as client:
|
|
223
|
+
|
|
224
|
+
async def worker(url: str) -> SiteStatus:
|
|
225
|
+
async with sem:
|
|
226
|
+
return await async_check_site(
|
|
227
|
+
url,
|
|
228
|
+
timeout=timeout,
|
|
229
|
+
retries=retries,
|
|
230
|
+
follow_redirects=follow_redirects,
|
|
231
|
+
max_redirects=max_redirects,
|
|
232
|
+
custom_headers=None, # headers already applied on client
|
|
233
|
+
verify_ssl=verify_ssl,
|
|
234
|
+
retry_delay=retry_delay,
|
|
235
|
+
client=client,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
return await asyncio.gather(*[worker(u) for u in sites])
|