klyrek-http 0.1.0__py3-none-any.whl

klyrek_http/__init__.py

@@ -0,0 +1,8 @@
+"""klyrek-http: scope-checked, rate-limited HTTP client for the Klyrek ecosystem."""
+
+from klyrek_http.client import KlyrekHTTPClient
+from klyrek_http.rate_limit import RateLimiter
+
+__version__ = "0.1.0"
+
+__all__ = ["KlyrekHTTPClient", "RateLimiter", "__version__"]

klyrek_http/client.py

@@ -0,0 +1,86 @@
+"""A scope-checked, rate-limited HTTP client shared across the Klyrek ecosystem."""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Any
+
+import httpx
+
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.logging import get_logger
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.rate_limit import RateLimiter
+
+logger = get_logger("http.client")
+
+
+class KlyrekHTTPClient:
+    """Every Klyrek package that talks to a target should issue requests through this client.
+
+    It enforces the scan's :class:`AuthorizationScope` and per-host rate limit on every
+    request, so no individual package has to remember to do so itself.
+    """
+
+    def __init__(
+        self,
+        scope: AuthorizationScope,
+        config: KlyrekConfig | None = None,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.scope = scope
+        self.config = config or KlyrekConfig()
+        min_interval = (
+            1.0 / self.config.rate_limit_per_host if self.config.rate_limit_per_host > 0 else 0.0
+        )
+        self.rate_limiter = RateLimiter(min_interval=min_interval)
+        self._client = httpx.Client(
+            headers={"User-Agent": self.config.user_agent},
+            timeout=self.config.timeout_seconds,
+            verify=self.config.verify_tls,
+            follow_redirects=True,
+            transport=transport,
+            event_hooks={"request": [self._enforce_scope_and_rate_limit]},
+        )
+
+    def _enforce_scope_and_rate_limit(self, request: httpx.Request) -> None:
+        # Registered as an httpx "request" hook rather than checked once up front, because
+        # follow_redirects=True makes httpx dispatch redirect hops as their own requests
+        # internally. A check only in request() would see the original URL and miss a
+        # redirect to an out-of-scope host.
+        url = str(request.url)
+        self.scope.check(url)
+        self.rate_limiter.wait(request.url.host)
+
+    def request(self, method: str, url: str, **kwargs: Any) -> httpx.Response:
+        logger.debug("%s %s", method, url)
+        return self._client.request(method, url, **kwargs)
+
+    def get(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("PUT", url, **kwargs)
+
+    def head(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("HEAD", url, **kwargs)
+
+    def delete(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("DELETE", url, **kwargs)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> KlyrekHTTPClient:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()

klyrek_http/rate_limit.py

@@ -0,0 +1,37 @@
+"""Per-host rate limiting so a scan never exceeds its authorized request rate."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+
+@dataclass
+class RateLimiter:
+    """Enforces a minimum interval between requests to the same host.
+
+    Hosts are tracked independently: waiting for ``a.com`` never blocks a
+    concurrent request to ``b.com``. ``clock``/``sleep`` are injectable so
+    tests can run instantly against a fake clock.
+    """
+
+    min_interval: float = 0.0
+    clock: Callable[[], float] = time.monotonic
+    sleep: Callable[[float], None] = time.sleep
+    _last_request_at: dict[str, float] = field(default_factory=dict, repr=False)
+
+    def wait(self, host: str) -> None:
+        """Block, if needed, until ``min_interval`` has passed since the last request to host."""
+        if self.min_interval <= 0:
+            self._last_request_at[host] = self.clock()
+            return
+
+        now = self.clock()
+        last = self._last_request_at.get(host)
+        if last is not None:
+            remaining = self.min_interval - (now - last)
+            if remaining > 0:
+                self.sleep(remaining)
+                now = self.clock()
+        self._last_request_at[host] = now

klyrek_http-0.1.0.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: klyrek-http
+Version: 0.1.0
+Summary: HTTP client abstraction and request management for the Klyrek ecosystem
+Author: Klyrek Contributors
+License: MIT
+Keywords: appsec,http,pentesting,reconnaissance,security
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Security
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: klyrek-core
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# klyrek-http
+
+The HTTP layer shared by every Klyrek package that talks to a target: `klyrek-crawler`,
+`klyrek-api`, `klyrek-tech`, `klyrek-auth`, `klyrek-js`, `klyrek-assets`, and `klyrek-headers` all
+issue requests through `KlyrekHTTPClient` rather than calling `httpx`/`requests` directly.
+
+Every request goes through two checks before it leaves the process:
+
+1. **Scope enforcement** — the URL's host must be declared in the scan's
+   [`AuthorizationScope`](../klyrek-core/README.md#scope-enforcement), or the request raises
+   `ScopeViolationError` instead of firing.
+2. **Rate limiting** — a per-host minimum interval (`KlyrekConfig.rate_limit_per_host`) keeps a
+   scan from hammering a target faster than was authorized.
+
+```python
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.client import KlyrekHTTPClient
+
+scope = AuthorizationScope(authorized_hosts=["target.com", "*.target.com"])
+
+with KlyrekHTTPClient(scope, config=KlyrekConfig(rate_limit_per_host=5)) as client:
+    response = client.get("https://target.com/")
+```

klyrek_http-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+klyrek_http/__init__.py,sha256=zfdB3REfyozkgkVr4JP8zzrkeAnHJDoeAo8zwGqcyQ0,266
+klyrek_http/client.py,sha256=n45IFg6HGrEphVNJBHQBFjru4V4ghnbcTlfhPYWhI94,3059
+klyrek_http/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+klyrek_http/rate_limit.py,sha256=wQ7Uy5ZpKu8qpN_Fjmu06Qz66ynTYiqVl6WRmyIVN0c,1280
+klyrek_http-0.1.0.dist-info/METADATA,sha256=OOcc-7xwHWFu9VBH-ERSR78n-F6GoPUrc00T3ZhomTY,1828
+klyrek_http-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+klyrek_http-0.1.0.dist-info/RECORD,,

klyrek_http-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any