klyrek-http 0.1.0__tar.gz

klyrek_http-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+.qodo/
+build/
+dist/
+.venv/
+venv/
+.env
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.log
+.idea/
+.vscode/
+!.vscode/extensions.json
+klyrek_output/

klyrek_http-0.1.0/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: klyrek-http
+Version: 0.1.0
+Summary: HTTP client abstraction and request management for the Klyrek ecosystem
+Author: Klyrek Contributors
+License: MIT
+Keywords: appsec,http,pentesting,reconnaissance,security
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Security
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: klyrek-core
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# klyrek-http
+
+The HTTP layer shared by every Klyrek package that talks to a target: `klyrek-crawler`,
+`klyrek-api`, `klyrek-tech`, `klyrek-auth`, `klyrek-js`, `klyrek-assets`, and `klyrek-headers` all
+issue requests through `KlyrekHTTPClient` rather than calling `httpx`/`requests` directly.
+
+Every request goes through two checks before it leaves the process:
+
+1. **Scope enforcement** — the URL's host must be declared in the scan's
+   [`AuthorizationScope`](../klyrek-core/README.md#scope-enforcement), or the request raises
+   `ScopeViolationError` instead of firing.
+2. **Rate limiting** — a per-host minimum interval (`KlyrekConfig.rate_limit_per_host`) keeps a
+   scan from hammering a target faster than was authorized.
+
+```python
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.client import KlyrekHTTPClient
+
+scope = AuthorizationScope(authorized_hosts=["target.com", "*.target.com"])
+
+with KlyrekHTTPClient(scope, config=KlyrekConfig(rate_limit_per_host=5)) as client:
+    response = client.get("https://target.com/")
+```

klyrek_http-0.1.0/README.md

@@ -0,0 +1,24 @@
+# klyrek-http
+
+The HTTP layer shared by every Klyrek package that talks to a target: `klyrek-crawler`,
+`klyrek-api`, `klyrek-tech`, `klyrek-auth`, `klyrek-js`, `klyrek-assets`, and `klyrek-headers` all
+issue requests through `KlyrekHTTPClient` rather than calling `httpx`/`requests` directly.
+
+Every request goes through two checks before it leaves the process:
+
+1. **Scope enforcement** — the URL's host must be declared in the scan's
+   [`AuthorizationScope`](../klyrek-core/README.md#scope-enforcement), or the request raises
+   `ScopeViolationError` instead of firing.
+2. **Rate limiting** — a per-host minimum interval (`KlyrekConfig.rate_limit_per_host`) keeps a
+   scan from hammering a target faster than was authorized.
+
+```python
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.client import KlyrekHTTPClient
+
+scope = AuthorizationScope(authorized_hosts=["target.com", "*.target.com"])
+
+with KlyrekHTTPClient(scope, config=KlyrekConfig(rate_limit_per_host=5)) as client:
+    response = client.get("https://target.com/")
+```

klyrek_http-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "klyrek-http"
+version = "0.1.0"
+description = "HTTP client abstraction and request management for the Klyrek ecosystem"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+authors = [{ name = "Klyrek Contributors" }]
+keywords = ["security", "reconnaissance", "appsec", "pentesting", "http"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Information Technology",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Security",
+]
+dependencies = [
+    "klyrek-core",
+    "httpx>=0.27",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "ruff>=0.6", "mypy>=1.10"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/klyrek_http"]

klyrek_http-0.1.0/src/klyrek_http/__init__.py

@@ -0,0 +1,8 @@
+"""klyrek-http: scope-checked, rate-limited HTTP client for the Klyrek ecosystem."""
+
+from klyrek_http.client import KlyrekHTTPClient
+from klyrek_http.rate_limit import RateLimiter
+
+__version__ = "0.1.0"
+
+__all__ = ["KlyrekHTTPClient", "RateLimiter", "__version__"]

klyrek_http-0.1.0/src/klyrek_http/client.py

@@ -0,0 +1,86 @@
+"""A scope-checked, rate-limited HTTP client shared across the Klyrek ecosystem."""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Any
+
+import httpx
+
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.logging import get_logger
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.rate_limit import RateLimiter
+
+logger = get_logger("http.client")
+
+
+class KlyrekHTTPClient:
+    """Every Klyrek package that talks to a target should issue requests through this client.
+
+    It enforces the scan's :class:`AuthorizationScope` and per-host rate limit on every
+    request, so no individual package has to remember to do so itself.
+    """
+
+    def __init__(
+        self,
+        scope: AuthorizationScope,
+        config: KlyrekConfig | None = None,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.scope = scope
+        self.config = config or KlyrekConfig()
+        min_interval = (
+            1.0 / self.config.rate_limit_per_host if self.config.rate_limit_per_host > 0 else 0.0
+        )
+        self.rate_limiter = RateLimiter(min_interval=min_interval)
+        self._client = httpx.Client(
+            headers={"User-Agent": self.config.user_agent},
+            timeout=self.config.timeout_seconds,
+            verify=self.config.verify_tls,
+            follow_redirects=True,
+            transport=transport,
+            event_hooks={"request": [self._enforce_scope_and_rate_limit]},
+        )
+
+    def _enforce_scope_and_rate_limit(self, request: httpx.Request) -> None:
+        # Registered as an httpx "request" hook rather than checked once up front, because
+        # follow_redirects=True makes httpx dispatch redirect hops as their own requests
+        # internally. A check only in request() would see the original URL and miss a
+        # redirect to an out-of-scope host.
+        url = str(request.url)
+        self.scope.check(url)
+        self.rate_limiter.wait(request.url.host)
+
+    def request(self, method: str, url: str, **kwargs: Any) -> httpx.Response:
+        logger.debug("%s %s", method, url)
+        return self._client.request(method, url, **kwargs)
+
+    def get(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("PUT", url, **kwargs)
+
+    def head(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("HEAD", url, **kwargs)
+
+    def delete(self, url: str, **kwargs: Any) -> httpx.Response:
+        return self.request("DELETE", url, **kwargs)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> KlyrekHTTPClient:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()

klyrek_http-0.1.0/src/klyrek_http/rate_limit.py

@@ -0,0 +1,37 @@
+"""Per-host rate limiting so a scan never exceeds its authorized request rate."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+
+@dataclass
+class RateLimiter:
+    """Enforces a minimum interval between requests to the same host.
+
+    Hosts are tracked independently: waiting for ``a.com`` never blocks a
+    concurrent request to ``b.com``. ``clock``/``sleep`` are injectable so
+    tests can run instantly against a fake clock.
+    """
+
+    min_interval: float = 0.0
+    clock: Callable[[], float] = time.monotonic
+    sleep: Callable[[float], None] = time.sleep
+    _last_request_at: dict[str, float] = field(default_factory=dict, repr=False)
+
+    def wait(self, host: str) -> None:
+        """Block, if needed, until ``min_interval`` has passed since the last request to host."""
+        if self.min_interval <= 0:
+            self._last_request_at[host] = self.clock()
+            return
+
+        now = self.clock()
+        last = self._last_request_at.get(host)
+        if last is not None:
+            remaining = self.min_interval - (now - last)
+            if remaining > 0:
+                self.sleep(remaining)
+                now = self.clock()
+        self._last_request_at[host] = now

klyrek_http-0.1.0/tests/test_client.py

@@ -0,0 +1,108 @@
+import httpx
+import pytest
+
+from klyrek_core.config import KlyrekConfig
+from klyrek_core.exceptions import ScopeViolationError
+from klyrek_core.scope import AuthorizationScope
+from klyrek_http.client import KlyrekHTTPClient
+
+
+def _echo_transport() -> httpx.MockTransport:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            json={"path": request.url.path, "ua": request.headers.get("user-agent")},
+        )
+
+    return httpx.MockTransport(handler)
+
+
+def test_request_within_scope_succeeds():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    config = KlyrekConfig(rate_limit_per_host=0)
+    with KlyrekHTTPClient(scope, config=config, transport=_echo_transport()) as client:
+        response = client.get("https://target.com/login")
+    assert response.status_code == 200
+    assert response.json()["path"] == "/login"
+
+
+def test_request_out_of_scope_raises():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    with KlyrekHTTPClient(scope, transport=_echo_transport()) as client, pytest.raises(
+        ScopeViolationError
+    ):
+        client.get("https://evil.com")
+
+
+def test_user_agent_applied_from_config():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    config = KlyrekConfig(user_agent="Klyrek-Test/1.0", rate_limit_per_host=0)
+    with KlyrekHTTPClient(scope, config=config, transport=_echo_transport()) as client:
+        response = client.get("https://target.com/")
+    assert response.json()["ua"] == "Klyrek-Test/1.0"
+
+
+def test_verbs_dispatch_correct_http_method():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    seen_methods: list[str] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen_methods.append(request.method)
+        return httpx.Response(200)
+
+    config = KlyrekConfig(rate_limit_per_host=0)
+    with KlyrekHTTPClient(scope, config=config, transport=httpx.MockTransport(handler)) as client:
+        client.get("https://target.com/")
+        client.post("https://target.com/")
+        client.put("https://target.com/")
+        client.head("https://target.com/")
+        client.delete("https://target.com/")
+
+    assert seen_methods == ["GET", "POST", "PUT", "HEAD", "DELETE"]
+
+
+def test_scope_violation_short_circuits_before_request():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    called = False
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal called
+        called = True
+        return httpx.Response(200)
+
+    with KlyrekHTTPClient(scope, transport=httpx.MockTransport(handler)) as client, pytest.raises(
+        ScopeViolationError
+    ):
+        client.get("https://evil.com")
+
+    assert called is False
+
+
+def test_redirect_to_out_of_scope_host_is_blocked():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    config = KlyrekConfig(rate_limit_per_host=0)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.host == "target.com":
+            return httpx.Response(302, headers={"Location": "https://evil.com/steal"})
+        return httpx.Response(200)  # pragma: no cover - should never be reached
+
+    with KlyrekHTTPClient(
+        scope, config=config, transport=httpx.MockTransport(handler)
+    ) as client, pytest.raises(ScopeViolationError):
+        client.get("https://target.com/login")
+
+
+def test_redirect_within_scope_is_followed():
+    scope = AuthorizationScope(authorized_hosts=["target.com"])
+    config = KlyrekConfig(rate_limit_per_host=0)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/old":
+            return httpx.Response(302, headers={"Location": "https://target.com/new"})
+        return httpx.Response(200, json={"path": request.url.path})
+
+    with KlyrekHTTPClient(scope, config=config, transport=httpx.MockTransport(handler)) as client:
+        response = client.get("https://target.com/old")
+
+    assert response.json()["path"] == "/new"

klyrek_http-0.1.0/tests/test_rate_limit.py

@@ -0,0 +1,67 @@
+import pytest
+
+from klyrek_http.rate_limit import RateLimiter
+
+
+class FakeClock:
+    """A controllable clock where sleep() advances time instead of blocking."""
+
+    def __init__(self, start: float = 0.0) -> None:
+        self.t = start
+        self.sleep_calls: list[float] = []
+
+    def now(self) -> float:
+        return self.t
+
+    def tick(self, seconds: float) -> None:
+        self.t += seconds
+
+    def advance(self, seconds: float) -> None:
+        self.sleep_calls.append(seconds)
+        self.t += seconds
+
+
+def _limiter(min_interval: float, clock: FakeClock) -> RateLimiter:
+    return RateLimiter(min_interval=min_interval, clock=clock.now, sleep=clock.advance)
+
+
+def test_no_wait_on_first_request():
+    clock = FakeClock()
+    limiter = _limiter(1.0, clock)
+    limiter.wait("target.com")
+    assert clock.sleep_calls == []
+
+
+def test_waits_remaining_interval():
+    clock = FakeClock()
+    limiter = _limiter(1.0, clock)
+    limiter.wait("target.com")
+    clock.tick(0.4)
+    limiter.wait("target.com")
+    assert clock.sleep_calls == [pytest.approx(0.6)]
+
+
+def test_no_wait_when_interval_already_elapsed():
+    clock = FakeClock()
+    limiter = _limiter(1.0, clock)
+    limiter.wait("target.com")
+    clock.tick(1.5)
+    limiter.wait("target.com")
+    assert clock.sleep_calls == []
+
+
+def test_hosts_tracked_independently():
+    clock = FakeClock()
+    limiter = _limiter(1.0, clock)
+    limiter.wait("a.com")
+    clock.tick(0.1)
+    limiter.wait("b.com")
+    assert clock.sleep_calls == []
+
+
+def test_zero_interval_never_waits():
+    clock = FakeClock()
+    limiter = _limiter(0.0, clock)
+    limiter.wait("target.com")
+    limiter.wait("target.com")
+    assert clock.sleep_calls == []