httpcheck 1.4.3__tar.gz → 1.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {httpcheck-1.4.3/httpcheck.egg-info → httpcheck-1.5.1}/PKG-INFO +83 -16
  2. httpcheck-1.4.3/PKG-INFO → httpcheck-1.5.1/README.md +64 -55
  3. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/__init__.py +4 -1
  4. httpcheck-1.5.1/httpcheck/async_site_checker.py +238 -0
  5. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/cli.py +103 -45
  6. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/common.py +4 -6
  7. httpcheck-1.5.1/httpcheck/config.py +143 -0
  8. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/file_handler.py +6 -7
  9. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/logger.py +2 -3
  10. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/output_formatter.py +0 -1
  11. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/site_checker.py +70 -38
  12. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/tld_manager.py +48 -25
  13. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/validation.py +12 -12
  14. httpcheck-1.4.3/README.md → httpcheck-1.5.1/httpcheck.egg-info/PKG-INFO +123 -11
  15. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/SOURCES.txt +4 -0
  16. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/requires.txt +16 -2
  17. {httpcheck-1.4.3 → httpcheck-1.5.1}/pyproject.toml +27 -6
  18. httpcheck-1.5.1/tests/test_async_site_checker.py +360 -0
  19. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_cli_integration.py +250 -28
  20. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_common.py +2 -2
  21. httpcheck-1.5.1/tests/test_config.py +323 -0
  22. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_httpcheck.py +1 -1
  23. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_init.py +2 -2
  24. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_performance.py +22 -4
  25. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_site_checker.py +151 -8
  26. httpcheck-1.5.1/tests/test_tld_manager.py +357 -0
  27. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_validation.py +5 -1
  28. httpcheck-1.4.3/tests/test_tld_manager.py +0 -101
  29. {httpcheck-1.4.3 → httpcheck-1.5.1}/LICENSE +0 -0
  30. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck/notification.py +0 -0
  31. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/dependency_links.txt +0 -0
  32. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/entry_points.txt +0 -0
  33. {httpcheck-1.4.3 → httpcheck-1.5.1}/httpcheck.egg-info/top_level.txt +0 -0
  34. {httpcheck-1.4.3 → httpcheck-1.5.1}/setup.cfg +0 -0
  35. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_file_handler.py +0 -0
  36. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_output_formats.py +0 -0
  37. {httpcheck-1.4.3 → httpcheck-1.5.1}/tests/test_request_customization.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: httpcheck
3
- Version: 1.4.3
3
+ Version: 1.5.1
4
4
  Summary: HTTP status code checker with advanced features
5
5
  Author-email: Thomas Juul Dyhr <thomas@dyhr.com>
6
6
  License: MIT
@@ -13,19 +13,27 @@ Classifier: Intended Audience :: System Administrators
13
13
  Classifier: License :: OSI Approved :: MIT License
14
14
  Classifier: Operating System :: OS Independent
15
15
  Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
16
  Classifier: Programming Language :: Python :: 3.10
18
17
  Classifier: Programming Language :: Python :: 3.11
19
18
  Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
20
21
  Classifier: Topic :: System :: Monitoring
21
22
  Classifier: Topic :: System :: Networking :: Monitoring
22
- Requires-Python: >=3.9
23
+ Requires-Python: >=3.10
23
24
  Description-Content-Type: text/markdown
24
25
  License-File: LICENSE
25
- Requires-Dist: requests>=2.32.5
26
+ Requires-Dist: requests>=2.33.0
27
+ Requires-Dist: httpx>=0.27.0
26
28
  Requires-Dist: tabulate>=0.9.0
27
29
  Requires-Dist: tqdm>=4.67.0
28
30
  Requires-Dist: validators>=0.22.0
31
+ Requires-Dist: tomli>=2.0.0; python_version < "3.11"
32
+ Requires-Dist: urllib3>=2.8.0
33
+ Requires-Dist: certifi>=2026.10.19
34
+ Requires-Dist: idna>=3.15
35
+ Requires-Dist: anyio>=4.14.2
36
+ Requires-Dist: h11>=0.16.0
29
37
  Provides-Extra: macos
30
38
  Requires-Dist: rumps>=0.4.0; extra == "macos"
31
39
  Requires-Dist: pyobjc-framework-Cocoa>=10.0; extra == "macos"
@@ -37,12 +45,27 @@ Requires-Dist: pytest>=7.0; extra == "dev"
37
45
  Requires-Dist: pytest-mock>=3.0; extra == "dev"
38
46
  Requires-Dist: pytest-cov>=4.0; extra == "dev"
39
47
  Requires-Dist: pytest-benchmark>=5.0; extra == "dev"
48
+ Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
40
49
  Requires-Dist: pylint>=3.1.0; extra == "dev"
41
- Requires-Dist: mypy>=1.0; extra == "dev"
50
+ Requires-Dist: mypy==2.4.0; extra == "dev"
51
+ Requires-Dist: types-requests>=2.31.0; extra == "dev"
52
+ Requires-Dist: types-tabulate>=0.9.0; extra == "dev"
53
+ Requires-Dist: types-tqdm>=4.66.0; extra == "dev"
54
+ Requires-Dist: tomli>=2.0.0; extra == "dev"
42
55
  Provides-Extra: build
43
56
  Requires-Dist: py2app>=0.28; extra == "build"
44
57
  Dynamic: license-file
45
58
 
59
+ <div align="right">
60
+
61
+ <a href="https://railway.com?referralCode=QhjuBc">
62
+
63
+ <img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
64
+
65
+ </a>
66
+
67
+ </div>
68
+
46
69
  [![PyPI version](https://badge.fury.io/py/httpcheck.svg)](https://badge.fury.io/py/httpcheck)
47
70
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/httpcheck)](https://pypi.org/project/httpcheck/)
48
71
  [![PyPI - Downloads](https://img.shields.io/pypi/dm/httpcheck)](https://pypi.org/project/httpcheck/)
@@ -51,15 +74,16 @@ Dynamic: license-file
51
74
  [![GitHub issues](https://img.shields.io/github/issues/docdyhr/httpcheck)](https://github.com/docdyhr/httpcheck/issues)
52
75
  ![GitHub repo size](https://img.shields.io/github/repo-size/docdyhr/httpcheck)
53
76
  ![GitHub](https://img.shields.io/github/license/docdyhr/httpcheck)
77
+ [![Docs](https://img.shields.io/badge/docs-readthedocs-brightgreen)](https://httpcheck.readthedocs.io/)
54
78
 
55
79
  # Check Websites HTTP Status Codes with httpcheck
56
80
 
57
81
  <img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
58
82
 
59
83
  * **Name**: httpcheck (CLI) / onSite (Menu Bar App)
60
- * **Current Version**: 1.4.3 (Enterprise-Grade Improvements)
61
- * **Target Version**: 1.5.0 (Performance & Configuration Features)
62
- * **Programming Language**: Python 3.9+
84
+ * **Current Version**: 1.5.1 (security patch for 1.5.0: Async I/O & Configuration)
85
+ * **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
86
+ * **Programming Language**: Python 3.10+
63
87
  * **Author**: Thomas Juul Dyhr
64
88
  * **Purpose**: Advanced HTTP status checker with monitoring capabilities
65
89
  * **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
@@ -67,9 +91,25 @@ Dynamic: license-file
67
91
 
68
92
  ## 🚀 Release Status
69
93
 
70
- **httpcheck v1.4.3 has been released! 🎉**
71
-
72
- ✅ **NEW in v1.4.3 (Published to PyPI):**
94
+ **httpcheck v1.5.1 has been released! 🎉**
95
+
96
+ ✅ **v1.5.1 (security patch):** installing httpcheck now also requires patched
97
+ versions of `urllib3`, `certifi`, `idna`, `anyio` and `h11`, so an install into
98
+ an existing environment cannot keep vulnerable copies. No behaviour changes.
99
+
100
+ ✅ **NEW in v1.5.0:**
101
+ - **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
102
+ `./.httpcheck.toml` (the project file overrides the user file); CLI flags always
103
+ take precedence
104
+ - **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
105
+ the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
106
+ faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
107
+ - **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
108
+ installing 1.4.3); Python 3.13 and 3.14 are tested in CI
109
+ - **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
110
+ work (1.4.3 reported the site as failed without retrying)
111
+
112
+ ✅ **v1.4.2-1.4.3 (Published to PyPI):**
73
113
  - **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
74
114
  - **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
75
115
  - **Professional Documentation**: 24-page documentation with ReadTheDocs theme
@@ -83,15 +123,13 @@ Dynamic: license-file
83
123
  - **Multiple Output Formats**: JSON and CSV support
84
124
  - **Advanced Request Control**: Custom headers, SSL options
85
125
 
86
- 🚀 **Next: v1.5.0 Development Focus**
87
- - **Async I/O**: 2-3x performance improvement for large site lists
88
- - **Configuration Files**: User-defined defaults and settings
126
+ 🔭 **Next: v1.6.0 (target Q1 2027)**
89
127
  - **Monitoring Mode**: Continuous site monitoring with notifications
90
128
  - **Enhanced UX**: Colored output and improved progress reporting
91
129
 
92
130
  See detailed plans in:
93
- - [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
94
- - [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
131
+ - [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
132
+ - [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
95
133
  - [TODO.md](TODO.md) - Current prioritized task list
96
134
  - [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
97
135
 
@@ -399,6 +437,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
399
437
  httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
400
438
  ```
401
439
 
440
+ ### configuration file
441
+
442
+ httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
443
+
444
+ ```toml
445
+ # ~/.httpcheck.toml
446
+
447
+ [defaults]
448
+ timeout = 10.0
449
+ retries = 3
450
+ follow_redirects = "always" # always | never | http-only | https-only
451
+ output_format = "table" # table | json | csv
452
+ verify_ssl = true
453
+ workers = 20
454
+ retry_delay = 0.5
455
+ max_redirects = 10
456
+
457
+ [headers]
458
+ User-Agent = "mybot/1.0"
459
+ Accept = "application/json"
460
+
461
+ [notifications]
462
+ enabled = true
463
+ on_failure = true
464
+ sound = "Ping"
465
+ ```
466
+
467
+ Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
468
+
402
469
  ### file input examples
403
470
 
404
471
  Create a domains file with comments and use it with httpcheck:
@@ -1,47 +1,12 @@
1
- Metadata-Version: 2.4
2
- Name: httpcheck
3
- Version: 1.4.3
4
- Summary: HTTP status code checker with advanced features
5
- Author-email: Thomas Juul Dyhr <thomas@dyhr.com>
6
- License: MIT
7
- Project-URL: Homepage, https://github.com/docdyhr/httpcheck
8
- Project-URL: Bug Tracker, https://github.com/docdyhr/httpcheck/issues
9
- Project-URL: Documentation, https://github.com/docdyhr/httpcheck/blob/master/README.md
10
- Classifier: Development Status :: 4 - Beta
11
- Classifier: Environment :: Console
12
- Classifier: Intended Audience :: System Administrators
13
- Classifier: License :: OSI Approved :: MIT License
14
- Classifier: Operating System :: OS Independent
15
- Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
- Classifier: Programming Language :: Python :: 3.10
18
- Classifier: Programming Language :: Python :: 3.11
19
- Classifier: Programming Language :: Python :: 3.12
20
- Classifier: Topic :: System :: Monitoring
21
- Classifier: Topic :: System :: Networking :: Monitoring
22
- Requires-Python: >=3.9
23
- Description-Content-Type: text/markdown
24
- License-File: LICENSE
25
- Requires-Dist: requests>=2.32.5
26
- Requires-Dist: tabulate>=0.9.0
27
- Requires-Dist: tqdm>=4.67.0
28
- Requires-Dist: validators>=0.22.0
29
- Provides-Extra: macos
30
- Requires-Dist: rumps>=0.4.0; extra == "macos"
31
- Requires-Dist: pyobjc-framework-Cocoa>=10.0; extra == "macos"
32
- Requires-Dist: pyobjc-framework-Foundation>=10.0; extra == "macos"
33
- Requires-Dist: pyobjc-framework-AppKit>=10.0; extra == "macos"
34
- Requires-Dist: pyobjc-core>=10.0; extra == "macos"
35
- Provides-Extra: dev
36
- Requires-Dist: pytest>=7.0; extra == "dev"
37
- Requires-Dist: pytest-mock>=3.0; extra == "dev"
38
- Requires-Dist: pytest-cov>=4.0; extra == "dev"
39
- Requires-Dist: pytest-benchmark>=5.0; extra == "dev"
40
- Requires-Dist: pylint>=3.1.0; extra == "dev"
41
- Requires-Dist: mypy>=1.0; extra == "dev"
42
- Provides-Extra: build
43
- Requires-Dist: py2app>=0.28; extra == "build"
44
- Dynamic: license-file
1
+ <div align="right">
2
+
3
+ <a href="https://railway.com?referralCode=QhjuBc">
4
+
5
+ <img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
6
+
7
+ </a>
8
+
9
+ </div>
45
10
 
46
11
  [![PyPI version](https://badge.fury.io/py/httpcheck.svg)](https://badge.fury.io/py/httpcheck)
47
12
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/httpcheck)](https://pypi.org/project/httpcheck/)
@@ -51,15 +16,16 @@ Dynamic: license-file
51
16
  [![GitHub issues](https://img.shields.io/github/issues/docdyhr/httpcheck)](https://github.com/docdyhr/httpcheck/issues)
52
17
  ![GitHub repo size](https://img.shields.io/github/repo-size/docdyhr/httpcheck)
53
18
  ![GitHub](https://img.shields.io/github/license/docdyhr/httpcheck)
19
+ [![Docs](https://img.shields.io/badge/docs-readthedocs-brightgreen)](https://httpcheck.readthedocs.io/)
54
20
 
55
21
  # Check Websites HTTP Status Codes with httpcheck
56
22
 
57
23
  <img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
58
24
 
59
25
  * **Name**: httpcheck (CLI) / onSite (Menu Bar App)
60
- * **Current Version**: 1.4.3 (Enterprise-Grade Improvements)
61
- * **Target Version**: 1.5.0 (Performance & Configuration Features)
62
- * **Programming Language**: Python 3.9+
26
+ * **Current Version**: 1.5.1 (security patch for 1.5.0: Async I/O & Configuration)
27
+ * **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
28
+ * **Programming Language**: Python 3.10+
63
29
  * **Author**: Thomas Juul Dyhr
64
30
  * **Purpose**: Advanced HTTP status checker with monitoring capabilities
65
31
  * **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
@@ -67,9 +33,25 @@ Dynamic: license-file
67
33
 
68
34
  ## 🚀 Release Status
69
35
 
70
- **httpcheck v1.4.3 has been released! 🎉**
71
-
72
- ✅ **NEW in v1.4.3 (Published to PyPI):**
36
+ **httpcheck v1.5.1 has been released! 🎉**
37
+
38
+ ✅ **v1.5.1 (security patch):** installing httpcheck now also requires patched
39
+ versions of `urllib3`, `certifi`, `idna`, `anyio` and `h11`, so an install into
40
+ an existing environment cannot keep vulnerable copies. No behaviour changes.
41
+
42
+ ✅ **NEW in v1.5.0:**
43
+ - **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
44
+ `./.httpcheck.toml` (the project file overrides the user file); CLI flags always
45
+ take precedence
46
+ - **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
47
+ the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
48
+ faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
49
+ - **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
50
+ installing 1.4.3); Python 3.13 and 3.14 are tested in CI
51
+ - **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
52
+ work (1.4.3 reported the site as failed without retrying)
53
+
54
+ ✅ **v1.4.2-1.4.3 (Published to PyPI):**
73
55
  - **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
74
56
  - **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
75
57
  - **Professional Documentation**: 24-page documentation with ReadTheDocs theme
@@ -83,15 +65,13 @@ Dynamic: license-file
83
65
  - **Multiple Output Formats**: JSON and CSV support
84
66
  - **Advanced Request Control**: Custom headers, SSL options
85
67
 
86
- 🚀 **Next: v1.5.0 Development Focus**
87
- - **Async I/O**: 2-3x performance improvement for large site lists
88
- - **Configuration Files**: User-defined defaults and settings
68
+ 🔭 **Next: v1.6.0 (target Q1 2027)**
89
69
  - **Monitoring Mode**: Continuous site monitoring with notifications
90
70
  - **Enhanced UX**: Colored output and improved progress reporting
91
71
 
92
72
  See detailed plans in:
93
- - [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
94
- - [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
73
+ - [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
74
+ - [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
95
75
  - [TODO.md](TODO.md) - Current prioritized task list
96
76
  - [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
97
77
 
@@ -399,6 +379,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
399
379
  httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
400
380
  ```
401
381
 
382
+ ### configuration file
383
+
384
+ httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
385
+
386
+ ```toml
387
+ # ~/.httpcheck.toml
388
+
389
+ [defaults]
390
+ timeout = 10.0
391
+ retries = 3
392
+ follow_redirects = "always" # always | never | http-only | https-only
393
+ output_format = "table" # table | json | csv
394
+ verify_ssl = true
395
+ workers = 20
396
+ retry_delay = 0.5
397
+ max_redirects = 10
398
+
399
+ [headers]
400
+ User-Agent = "mybot/1.0"
401
+ Accept = "application/json"
402
+
403
+ [notifications]
404
+ enabled = true
405
+ on_failure = true
406
+ sound = "Ping"
407
+ ```
408
+
409
+ Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
410
+
402
411
  ### file input examples
403
412
 
404
413
  Create a domains file with comments and use it with httpcheck:
@@ -5,13 +5,14 @@ A Python CLI tool for checking HTTP status codes of websites with
5
5
  threading, redirect handling, TLD validation, and system notifications.
6
6
  """
7
7
 
8
- __version__ = "1.4.3"
8
+ __version__ = "1.5.1"
9
9
  __author__ = "Thomas Juul Dyhr"
10
10
  __email__ = "thomas@dyhr.com"
11
11
 
12
12
  # Import specific items for public API
13
13
  from .cli import main
14
14
  from .common import VERSION, InvalidTLDException, SiteStatus
15
+ from .config import DEFAULT_CONFIG, load_config
15
16
  from .file_handler import FileInputHandler, url_validation
16
17
  from .notification import notify
17
18
  from .output_formatter import (
@@ -37,6 +38,8 @@ from .validation import (
37
38
  __all__ = [
38
39
  # Core functionality
39
40
  "check_site",
41
+ "load_config",
42
+ "DEFAULT_CONFIG",
40
43
  "print_format",
41
44
  "format_json",
42
45
  "format_csv",
@@ -0,0 +1,238 @@
1
+ """Async HTTP site checking for httpcheck."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import time
7
+ from collections.abc import Iterable
8
+ from urllib.parse import urlparse
9
+
10
+ import httpx
11
+
12
+ from .common import STATUS_CODES, VERSION, SiteStatus
13
+
14
+ DEFAULT_LIMITS = httpx.Limits(max_connections=50, max_keepalive_connections=20)
15
+
16
+
17
+ def _create_headers(custom_headers: dict | None = None) -> dict:
18
+ """Create default headers including User-Agent."""
19
+ headers = {"User-Agent": f"httpcheck Async Agent {VERSION}"}
20
+ if custom_headers:
21
+ headers.update(custom_headers)
22
+ return headers
23
+
24
+
25
+ def _extract_domain(url_like: str | httpx.URL, fallback: str) -> str:
26
+ """Return host component from httpx.URL or string URL."""
27
+ try:
28
+ if isinstance(url_like, httpx.URL):
29
+ return str(url_like.host)
30
+ parsed = urlparse(str(url_like))
31
+ return parsed.hostname or fallback
32
+ except (AttributeError, ValueError):
33
+ return fallback
34
+
35
+
36
+ def _extract_redirects(
37
+ response: httpx.Response,
38
+ ) -> tuple[list[tuple[str, int]], list[tuple[str, int, float]]]:
39
+ """Return redirect chain and timing from an httpx response."""
40
+ chain: list[tuple[str, int]] = []
41
+ timing: list[tuple[str, int, float]] = []
42
+ for hop in response.history:
43
+ chain.append((str(hop.url), hop.status_code))
44
+ timing.append((str(hop.url), hop.status_code, 0.0))
45
+ chain.append((str(response.url), response.status_code))
46
+ timing.append((str(response.url), response.status_code, 0.0))
47
+ return chain, timing
48
+
49
+
50
+ def _should_stop_redirect(follow_redirects: str, redirect_url: httpx.URL) -> bool:
51
+ """Return True when protocol policy blocks following this redirect."""
52
+ if follow_redirects == "http-only" and redirect_url.scheme == "https":
53
+ return True
54
+ if follow_redirects == "https-only" and redirect_url.scheme == "http":
55
+ return True
56
+ return False
57
+
58
+
59
+ async def _manual_redirect_flow(
60
+ client: httpx.AsyncClient,
61
+ url: str,
62
+ *,
63
+ timeout: float,
64
+ max_redirects: int,
65
+ follow_redirects: str,
66
+ ) -> tuple[httpx.Response, float, list[tuple[str, int]], list[tuple[str, int, float]]]:
67
+ """Follow redirects manually to enforce protocol restrictions."""
68
+ chain: list[tuple[str, int]] = []
69
+ timing: list[tuple[str, int, float]] = []
70
+
71
+ current_url: httpx.URL = httpx.URL(url)
72
+ total_start = time.monotonic()
73
+ redirects = 0
74
+ while True:
75
+ hop_start = time.monotonic()
76
+ response = await client.get(
77
+ str(current_url),
78
+ follow_redirects=False,
79
+ timeout=timeout,
80
+ )
81
+ hop_elapsed = time.monotonic() - hop_start
82
+
83
+ chain.append((str(response.url), response.status_code))
84
+ timing.append((str(response.url), response.status_code, hop_elapsed))
85
+
86
+ if not response.is_redirect or "location" not in response.headers:
87
+ break
88
+
89
+ if redirects >= max_redirects:
90
+ break
91
+
92
+ location = response.headers.get("location", "")
93
+ base_url = (
94
+ response.url
95
+ if isinstance(response.url, httpx.URL)
96
+ else httpx.URL(str(response.url))
97
+ )
98
+ next_url = base_url.join(location)
99
+ if _should_stop_redirect(follow_redirects, next_url):
100
+ break
101
+
102
+ redirects += 1
103
+ current_url = next_url
104
+
105
+ total_elapsed = time.monotonic() - total_start
106
+ return response, total_elapsed, chain, timing
107
+
108
+
109
+ async def async_check_site(
110
+ site: str,
111
+ timeout: float = 5.0,
112
+ retries: int = 2,
113
+ *,
114
+ follow_redirects: str = "always",
115
+ max_redirects: int = 30,
116
+ custom_headers: dict | None = None,
117
+ verify_ssl: bool = True,
118
+ retry_delay: float = 1.0,
119
+ client: httpx.AsyncClient | None = None,
120
+ ) -> SiteStatus:
121
+ """Async version of check_site with optional protocol-restricted redirects."""
122
+ headers = _create_headers(custom_headers)
123
+ allow_redirects = follow_redirects == "always"
124
+ manual_redirects = follow_redirects in {"http-only", "https-only"}
125
+
126
+ async def _execute_request(
127
+ active_client: httpx.AsyncClient,
128
+ ) -> tuple[httpx.Response, float]:
129
+ start = time.monotonic()
130
+ response = await active_client.get(
131
+ site,
132
+ headers=headers,
133
+ timeout=timeout,
134
+ follow_redirects=allow_redirects,
135
+ )
136
+ elapsed = time.monotonic() - start
137
+ return response, elapsed
138
+
139
+ # httpx sets max_redirects per client, not per request: a caller-supplied
140
+ # client keeps its own limit.
141
+ owns_client = client is None
142
+ active_client = client or httpx.AsyncClient(
143
+ verify=verify_ssl,
144
+ limits=DEFAULT_LIMITS,
145
+ headers=headers,
146
+ max_redirects=max_redirects,
147
+ )
148
+
149
+ try:
150
+ for attempt in range(retries + 1):
151
+ try:
152
+ if manual_redirects:
153
+ (
154
+ response,
155
+ response_time,
156
+ redirect_chain,
157
+ redirect_timing,
158
+ ) = await _manual_redirect_flow(
159
+ active_client,
160
+ site,
161
+ timeout=timeout,
162
+ max_redirects=max_redirects,
163
+ follow_redirects=follow_redirects,
164
+ )
165
+ else:
166
+ response, response_time = await _execute_request(active_client)
167
+ redirect_chain, redirect_timing = _extract_redirects(response)
168
+
169
+ domain = _extract_domain(getattr(response, "url", site), site)
170
+ return SiteStatus(
171
+ domain=domain,
172
+ status=str(response.status_code),
173
+ message=STATUS_CODES.get(str(response.status_code), "Unknown"),
174
+ redirect_chain=redirect_chain,
175
+ response_time=response_time,
176
+ redirect_timing=redirect_timing,
177
+ )
178
+ except (httpx.RequestError, httpx.HTTPError, ValueError, OSError) as exc:
179
+ if attempt == retries:
180
+ status = (
181
+ "[timeout]"
182
+ if isinstance(exc, httpx.TimeoutException)
183
+ else "[connection error]"
184
+ )
185
+ domain = site
186
+ if isinstance(exc, httpx.HTTPError):
187
+ try:
188
+ # .request raises RuntimeError when it was never set.
189
+ domain = _extract_domain(exc.request.url, site)
190
+ except RuntimeError:
191
+ pass
192
+ return SiteStatus(domain, status, str(exc))
193
+ if retry_delay > 0:
194
+ await asyncio.sleep(retry_delay)
195
+ # Only reached when retries < 0, as in the sync check_site.
196
+ return SiteStatus(site, "[unknown error]", "All retries failed")
197
+ finally:
198
+ if owns_client:
199
+ await active_client.aclose()
200
+
201
+
202
+ async def async_check_sites(
203
+ sites: Iterable[str],
204
+ *,
205
+ timeout: float = 5.0,
206
+ retries: int = 2,
207
+ follow_redirects: str = "always",
208
+ max_redirects: int = 30,
209
+ custom_headers: dict | None = None,
210
+ verify_ssl: bool = True,
211
+ retry_delay: float = 1.0,
212
+ concurrency: int = 50,
213
+ ) -> list[SiteStatus]:
214
+ """Check many sites concurrently with a shared client and semaphore."""
215
+ sem = asyncio.Semaphore(concurrency)
216
+ headers = _create_headers(custom_headers)
217
+ async with httpx.AsyncClient(
218
+ verify=verify_ssl,
219
+ limits=DEFAULT_LIMITS,
220
+ headers=headers,
221
+ max_redirects=max_redirects,
222
+ ) as client:
223
+
224
+ async def worker(url: str) -> SiteStatus:
225
+ async with sem:
226
+ return await async_check_site(
227
+ url,
228
+ timeout=timeout,
229
+ retries=retries,
230
+ follow_redirects=follow_redirects,
231
+ max_redirects=max_redirects,
232
+ custom_headers=None, # headers already applied on client
233
+ verify_ssl=verify_ssl,
234
+ retry_delay=retry_delay,
235
+ client=client,
236
+ )
237
+
238
+ return await asyncio.gather(*[worker(u) for u in sites])