httpcheck 1.4.3__tar.gz → 1.5.0__tar.gz

{httpcheck-1.4.3 → httpcheck-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpcheck
-Version: 1.4.3
+Version: 1.5.0
 Summary: HTTP status code checker with advanced features
 Author-email: Thomas Juul Dyhr <thomas@dyhr.com>
 License: MIT
@@ -13,19 +13,22 @@ Classifier: Intended Audience :: System Administrators
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: System :: Monitoring
 Classifier: Topic :: System :: Networking :: Monitoring
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests>=2.32.5
+Requires-Dist: requests>=2.33.0
+Requires-Dist: httpx>=0.27.0
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: tqdm>=4.67.0
 Requires-Dist: validators>=0.22.0
+Requires-Dist: tomli>=2.0.0; python_version < "3.11"
 Provides-Extra: macos
 Requires-Dist: rumps>=0.4.0; extra == "macos"
 Requires-Dist: pyobjc-framework-Cocoa>=10.0; extra == "macos"
@@ -37,12 +40,27 @@ Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-mock>=3.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
 Requires-Dist: pytest-benchmark>=5.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
 Requires-Dist: pylint>=3.1.0; extra == "dev"
-Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: mypy==1.20.2; extra == "dev"
+Requires-Dist: types-requests>=2.31.0; extra == "dev"
+Requires-Dist: types-tabulate>=0.9.0; extra == "dev"
+Requires-Dist: types-tqdm>=4.66.0; extra == "dev"
+Requires-Dist: tomli>=2.0.0; extra == "dev"
 Provides-Extra: build
 Requires-Dist: py2app>=0.28; extra == "build"
 Dynamic: license-file
 
+<div align="right">
+
+<a href="https://railway.com?referralCode=QhjuBc">
+
+  <img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
+
+</a>
+
+</div>
+
 [![PyPI version](https://badge.fury.io/py/httpcheck.svg)](https://badge.fury.io/py/httpcheck)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/httpcheck)](https://pypi.org/project/httpcheck/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/httpcheck)](https://pypi.org/project/httpcheck/)
@@ -51,15 +69,16 @@ Dynamic: license-file
 [![GitHub issues](https://img.shields.io/github/issues/docdyhr/httpcheck)](https://github.com/docdyhr/httpcheck/issues)
 ![GitHub repo size](https://img.shields.io/github/repo-size/docdyhr/httpcheck)
 ![GitHub](https://img.shields.io/github/license/docdyhr/httpcheck)
+[![Docs](https://img.shields.io/badge/docs-readthedocs-brightgreen)](https://httpcheck.readthedocs.io/)
 
 # Check Websites HTTP Status Codes with httpcheck
 
 <img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
 
 * **Name**: httpcheck (CLI) / onSite (Menu Bar App)
-* **Current Version**: 1.4.3 (Enterprise-Grade Improvements)
-* **Target Version**: 1.5.0 (Performance & Configuration Features)
-* **Programming Language**: Python 3.9+
+* **Current Version**: 1.5.0 (Async I/O & Configuration)
+* **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
+* **Programming Language**: Python 3.10+
 * **Author**: Thomas Juul Dyhr
 * **Purpose**: Advanced HTTP status checker with monitoring capabilities
 * **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
@@ -67,9 +86,21 @@ Dynamic: license-file
 
 ## 🚀 Release Status
 
-**httpcheck v1.4.3 has been released! 🎉**
-
-✅ **NEW in v1.4.3 (Published to PyPI):**
+**httpcheck v1.5.0 has been released! 🎉**
+
+✅ **NEW in v1.5.0:**
+- **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
+  `./.httpcheck.toml` (the project file overrides the user file); CLI flags always
+  take precedence
+- **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
+  the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
+  faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
+- **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
+  installing 1.4.3); Python 3.13 and 3.14 are tested in CI
+- **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
+  work (1.4.3 reported the site as failed without retrying)
+
+✅ **v1.4.2-1.4.3 (Published to PyPI):**
 - **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
 - **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
 - **Professional Documentation**: 24-page documentation with ReadTheDocs theme
@@ -83,15 +114,13 @@ Dynamic: license-file
 - **Multiple Output Formats**: JSON and CSV support
 - **Advanced Request Control**: Custom headers, SSL options
 
-🚀 **Next: v1.5.0 Development Focus**
-- **Async I/O**: 2-3x performance improvement for large site lists
-- **Configuration Files**: User-defined defaults and settings
+🔭 **Next: v1.6.0 (target Q1 2027)**
 - **Monitoring Mode**: Continuous site monitoring with notifications
 - **Enhanced UX**: Colored output and improved progress reporting
 
 See detailed plans in:
-- [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
-- [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
+- [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
+- [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
 - [TODO.md](TODO.md) - Current prioritized task list
 - [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
 
@@ -399,6 +428,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
 httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
 ```
 
+### configuration file
+
+httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
+
+```toml
+# ~/.httpcheck.toml
+
+[defaults]
+timeout = 10.0
+retries = 3
+follow_redirects = "always"   # always | never | http-only | https-only
+output_format = "table"       # table | json | csv
+verify_ssl = true
+workers = 20
+retry_delay = 0.5
+max_redirects = 10
+
+[headers]
+User-Agent = "mybot/1.0"
+Accept = "application/json"
+
+[notifications]
+enabled = true
+on_failure = true
+sound = "Ping"
+```
+
+Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
+
 ### file input examples
 
 Create a domains file with comments and use it with httpcheck:

{httpcheck-1.4.3 → httpcheck-1.5.0}/README.md

@@ -1,3 +1,13 @@
+<div align="right">
+
+<a href="https://railway.com?referralCode=QhjuBc">
+
+  <img width="160" src="https://raw.githubusercontent.com/docdyhr/.github/main/assets/railway-corner-v2@2x.png" alt="Deploy on Railway — $20 free credits">
+
+</a>
+
+</div>
+
 [![PyPI version](https://badge.fury.io/py/httpcheck.svg)](https://badge.fury.io/py/httpcheck)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/httpcheck)](https://pypi.org/project/httpcheck/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/httpcheck)](https://pypi.org/project/httpcheck/)
@@ -6,15 +16,16 @@
 [![GitHub issues](https://img.shields.io/github/issues/docdyhr/httpcheck)](https://github.com/docdyhr/httpcheck/issues)
 ![GitHub repo size](https://img.shields.io/github/repo-size/docdyhr/httpcheck)
 ![GitHub](https://img.shields.io/github/license/docdyhr/httpcheck)
+[![Docs](https://img.shields.io/badge/docs-readthedocs-brightgreen)](https://httpcheck.readthedocs.io/)
 
 # Check Websites HTTP Status Codes with httpcheck
 
 <img src="images/onSiteLogo.png" alt="onSite Logo" width="50%">
 
 * **Name**: httpcheck (CLI) / onSite (Menu Bar App)
-* **Current Version**: 1.4.3 (Enterprise-Grade Improvements)
-* **Target Version**: 1.5.0 (Performance & Configuration Features)
-* **Programming Language**: Python 3.9+
+* **Current Version**: 1.5.0 (Async I/O & Configuration)
+* **Next Version**: 1.6.0 (Monitoring Mode, target Q1 2027)
+* **Programming Language**: Python 3.10+
 * **Author**: Thomas Juul Dyhr
 * **Purpose**: Advanced HTTP status checker with monitoring capabilities
 * **Development Status**: Active - See [ROADMAP.md](ROADMAP.md) for development plan
@@ -22,9 +33,21 @@
 
 ## 🚀 Release Status
 
-**httpcheck v1.4.3 has been released! 🎉**
-
-✅ **NEW in v1.4.3 (Published to PyPI):**
+**httpcheck v1.5.0 has been released! 🎉**
+
+✅ **NEW in v1.5.0:**
+- **Configuration Files**: User-defined defaults in `~/.httpcheck.toml` and
+  `./.httpcheck.toml` (the project file overrides the user file); CLI flags always
+  take precedence
+- **Async I/O** (experimental): `--async` uses an httpx-based concurrent path. In
+  the [benchmark against v1.4.3](docs/development/ASYNC_BENCHMARK.md) it is not
+  faster than `-f` at the same `--workers`; for speed, raise `-f --workers`
+- **Python 3.10+**: v1.5.0 drops Python 3.9 support (pip on Python 3.9 keeps
+  installing 1.4.3); Python 3.13 and 3.14 are tested in CI
+- **Retry fix**: with `--follow-redirects http-only` or `https-only`, retries now
+  work (1.4.3 reported the site as failed without retrying)
+
+✅ **v1.4.2-1.4.3 (Published to PyPI):**
 - **Structured Logging**: --debug, --log-file, --log-json flags for production monitoring
 - **Comprehensive Testing**: 297 tests with 88% coverage (CLI: 94%)
 - **Professional Documentation**: 24-page documentation with ReadTheDocs theme
@@ -38,15 +61,13 @@
 - **Multiple Output Formats**: JSON and CSV support
 - **Advanced Request Control**: Custom headers, SSL options
 
-🚀 **Next: v1.5.0 Development Focus**
-- **Async I/O**: 2-3x performance improvement for large site lists
-- **Configuration Files**: User-defined defaults and settings
+🔭 **Next: v1.6.0 (target Q1 2027)**
 - **Monitoring Mode**: Continuous site monitoring with notifications
 - **Enhanced UX**: Colored output and improved progress reporting
 
 See detailed plans in:
-- [DEVELOPMENT_PLAN.md](DEVELOPMENT_PLAN.md) - Executive overview and timeline
-- [DEVELOPMENT.md](DEVELOPMENT.md) - Technical implementation guide
+- [DEVELOPMENT_PLAN.md](docs/development/DEVELOPMENT_PLAN.md) - Executive overview and timeline
+- [DEVELOPMENT.md](docs/development/DEVELOPMENT.md) - Technical implementation guide
 - [TODO.md](TODO.md) - Current prioritized task list
 - [ROADMAP.md](ROADMAP.md) - Long-term vision through v2.0.0
 
@@ -354,6 +375,35 @@ httpcheck --timeout 30 --retries 5 --retry-delay 2.0 slow.example.com
 httpcheck -H "API-Key: secret" --output json --no-verify-ssl https://api.example.com
 ```
 
+### configuration file
+
+httpcheck reads defaults from `~/.httpcheck.toml` (user-wide) and/or `./.httpcheck.toml` (project-local). Project-level settings override user-level settings; any CLI flag always takes final precedence.
+
+```toml
+# ~/.httpcheck.toml
+
+[defaults]
+timeout = 10.0
+retries = 3
+follow_redirects = "always"   # always | never | http-only | https-only
+output_format = "table"       # table | json | csv
+verify_ssl = true
+workers = 20
+retry_delay = 0.5
+max_redirects = 10
+
+[headers]
+User-Agent = "mybot/1.0"
+Accept = "application/json"
+
+[notifications]
+enabled = true
+on_failure = true
+sound = "Ping"
+```
+
+Unknown keys are silently ignored with a `UserWarning`. Invalid TOML falls back to no config (also a `UserWarning`).
+
 ### file input examples
 
 Create a domains file with comments and use it with httpcheck:

{httpcheck-1.4.3 → httpcheck-1.5.0}/httpcheck/__init__.py

@@ -5,13 +5,14 @@ A Python CLI tool for checking HTTP status codes of websites with
 threading, redirect handling, TLD validation, and system notifications.
 """
 
-__version__ = "1.4.3"
+__version__ = "1.5.0"
 __author__ = "Thomas Juul Dyhr"
 __email__ = "thomas@dyhr.com"
 
 # Import specific items for public API
 from .cli import main
 from .common import VERSION, InvalidTLDException, SiteStatus
+from .config import DEFAULT_CONFIG, load_config
 from .file_handler import FileInputHandler, url_validation
 from .notification import notify
 from .output_formatter import (
@@ -37,6 +38,8 @@ from .validation import (
 __all__ = [
     # Core functionality
     "check_site",
+    "load_config",
+    "DEFAULT_CONFIG",
     "print_format",
     "format_json",
     "format_csv",

httpcheck-1.5.0/httpcheck/async_site_checker.py

@@ -0,0 +1,238 @@
+"""Async HTTP site checking for httpcheck."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Iterable
+from urllib.parse import urlparse
+
+import httpx
+
+from .common import STATUS_CODES, VERSION, SiteStatus
+
+DEFAULT_LIMITS = httpx.Limits(max_connections=50, max_keepalive_connections=20)
+
+
+def _create_headers(custom_headers: dict | None = None) -> dict:
+    """Create default headers including User-Agent."""
+    headers = {"User-Agent": f"httpcheck Async Agent {VERSION}"}
+    if custom_headers:
+        headers.update(custom_headers)
+    return headers
+
+
+def _extract_domain(url_like: str | httpx.URL, fallback: str) -> str:
+    """Return host component from httpx.URL or string URL."""
+    try:
+        if isinstance(url_like, httpx.URL):
+            return str(url_like.host)
+        parsed = urlparse(str(url_like))
+        return parsed.hostname or fallback
+    except (AttributeError, ValueError):
+        return fallback
+
+
+def _extract_redirects(
+    response: httpx.Response,
+) -> tuple[list[tuple[str, int]], list[tuple[str, int, float]]]:
+    """Return redirect chain and timing from an httpx response."""
+    chain: list[tuple[str, int]] = []
+    timing: list[tuple[str, int, float]] = []
+    for hop in response.history:
+        chain.append((str(hop.url), hop.status_code))
+        timing.append((str(hop.url), hop.status_code, 0.0))
+    chain.append((str(response.url), response.status_code))
+    timing.append((str(response.url), response.status_code, 0.0))
+    return chain, timing
+
+
+def _should_stop_redirect(follow_redirects: str, redirect_url: httpx.URL) -> bool:
+    """Return True when protocol policy blocks following this redirect."""
+    if follow_redirects == "http-only" and redirect_url.scheme == "https":
+        return True
+    if follow_redirects == "https-only" and redirect_url.scheme == "http":
+        return True
+    return False
+
+
+async def _manual_redirect_flow(
+    client: httpx.AsyncClient,
+    url: str,
+    *,
+    timeout: float,
+    max_redirects: int,
+    follow_redirects: str,
+) -> tuple[httpx.Response, float, list[tuple[str, int]], list[tuple[str, int, float]]]:
+    """Follow redirects manually to enforce protocol restrictions."""
+    chain: list[tuple[str, int]] = []
+    timing: list[tuple[str, int, float]] = []
+
+    current_url: httpx.URL = httpx.URL(url)
+    total_start = time.monotonic()
+    redirects = 0
+    while True:
+        hop_start = time.monotonic()
+        response = await client.get(
+            str(current_url),
+            follow_redirects=False,
+            timeout=timeout,
+        )
+        hop_elapsed = time.monotonic() - hop_start
+
+        chain.append((str(response.url), response.status_code))
+        timing.append((str(response.url), response.status_code, hop_elapsed))
+
+        if not response.is_redirect or "location" not in response.headers:
+            break
+
+        if redirects >= max_redirects:
+            break
+
+        location = response.headers.get("location", "")
+        base_url = (
+            response.url
+            if isinstance(response.url, httpx.URL)
+            else httpx.URL(str(response.url))
+        )
+        next_url = base_url.join(location)
+        if _should_stop_redirect(follow_redirects, next_url):
+            break
+
+        redirects += 1
+        current_url = next_url
+
+    total_elapsed = time.monotonic() - total_start
+    return response, total_elapsed, chain, timing
+
+
+async def async_check_site(
+    site: str,
+    timeout: float = 5.0,
+    retries: int = 2,
+    *,
+    follow_redirects: str = "always",
+    max_redirects: int = 30,
+    custom_headers: dict | None = None,
+    verify_ssl: bool = True,
+    retry_delay: float = 1.0,
+    client: httpx.AsyncClient | None = None,
+) -> SiteStatus:
+    """Async version of check_site with optional protocol-restricted redirects."""
+    headers = _create_headers(custom_headers)
+    allow_redirects = follow_redirects == "always"
+    manual_redirects = follow_redirects in {"http-only", "https-only"}
+
+    async def _execute_request(
+        active_client: httpx.AsyncClient,
+    ) -> tuple[httpx.Response, float]:
+        start = time.monotonic()
+        response = await active_client.get(
+            site,
+            headers=headers,
+            timeout=timeout,
+            follow_redirects=allow_redirects,
+        )
+        elapsed = time.monotonic() - start
+        return response, elapsed
+
+    # httpx sets max_redirects per client, not per request: a caller-supplied
+    # client keeps its own limit.
+    owns_client = client is None
+    active_client = client or httpx.AsyncClient(
+        verify=verify_ssl,
+        limits=DEFAULT_LIMITS,
+        headers=headers,
+        max_redirects=max_redirects,
+    )
+
+    try:
+        for attempt in range(retries + 1):
+            try:
+                if manual_redirects:
+                    (
+                        response,
+                        response_time,
+                        redirect_chain,
+                        redirect_timing,
+                    ) = await _manual_redirect_flow(
+                        active_client,
+                        site,
+                        timeout=timeout,
+                        max_redirects=max_redirects,
+                        follow_redirects=follow_redirects,
+                    )
+                else:
+                    response, response_time = await _execute_request(active_client)
+                    redirect_chain, redirect_timing = _extract_redirects(response)
+
+                domain = _extract_domain(getattr(response, "url", site), site)
+                return SiteStatus(
+                    domain=domain,
+                    status=str(response.status_code),
+                    message=STATUS_CODES.get(str(response.status_code), "Unknown"),
+                    redirect_chain=redirect_chain,
+                    response_time=response_time,
+                    redirect_timing=redirect_timing,
+                )
+            except (httpx.RequestError, httpx.HTTPError, ValueError, OSError) as exc:
+                if attempt == retries:
+                    status = (
+                        "[timeout]"
+                        if isinstance(exc, httpx.TimeoutException)
+                        else "[connection error]"
+                    )
+                    domain = site
+                    if isinstance(exc, httpx.HTTPError):
+                        try:
+                            # .request raises RuntimeError when it was never set.
+                            domain = _extract_domain(exc.request.url, site)
+                        except RuntimeError:
+                            pass
+                    return SiteStatus(domain, status, str(exc))
+                if retry_delay > 0:
+                    await asyncio.sleep(retry_delay)
+        # Only reached when retries < 0, as in the sync check_site.
+        return SiteStatus(site, "[unknown error]", "All retries failed")
+    finally:
+        if owns_client:
+            await active_client.aclose()
+
+
+async def async_check_sites(
+    sites: Iterable[str],
+    *,
+    timeout: float = 5.0,
+    retries: int = 2,
+    follow_redirects: str = "always",
+    max_redirects: int = 30,
+    custom_headers: dict | None = None,
+    verify_ssl: bool = True,
+    retry_delay: float = 1.0,
+    concurrency: int = 50,
+) -> list[SiteStatus]:
+    """Check many sites concurrently with a shared client and semaphore."""
+    sem = asyncio.Semaphore(concurrency)
+    headers = _create_headers(custom_headers)
+    async with httpx.AsyncClient(
+        verify=verify_ssl,
+        limits=DEFAULT_LIMITS,
+        headers=headers,
+        max_redirects=max_redirects,
+    ) as client:
+
+        async def worker(url: str) -> SiteStatus:
+            async with sem:
+                return await async_check_site(
+                    url,
+                    timeout=timeout,
+                    retries=retries,
+                    follow_redirects=follow_redirects,
+                    max_redirects=max_redirects,
+                    custom_headers=None,  # headers already applied on client
+                    verify_ssl=verify_ssl,
+                    retry_delay=retry_delay,
+                    client=client,
+                )
+
+        return await asyncio.gather(*[worker(u) for u in sites])