neptls 0.1.0__py3-none-any.whl

neptls/__init__.py

@@ -0,0 +1,52 @@
+"""NepTLS: an HTTP client and TLS research toolkit.
+
+Developed by Diwas Khatri (@diwaskhatri07).
+"""
+
+from .async_client import AsyncClient
+from .client import Client, delete, get, head, options, patch, post, put, request
+from .diagnostics import diagnose, inspect
+from .exceptions import HTTPStatusError, NepTLSError, RequestError, TimeoutError
+from .fingerprints import ClientFingerprint
+from .models import RequestTiming, Response
+from .profiles import BrowserProfile, PROFILES, get_profile
+from .proxy import Proxy
+from .tls import TLSConfig, TLSFingerprint
+from . import crypto, fingerprints as fingerprint, profiles, pow, user_agents
+from .user_agents import ua
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AsyncClient",
+    "BrowserProfile",
+    "Client",
+    "ClientFingerprint",
+    "HTTPStatusError",
+    "NepTLSError",
+    "PROFILES",
+    "Proxy",
+    "RequestError",
+    "RequestTiming",
+    "Response",
+    "TLSConfig",
+    "TLSFingerprint",
+    "TimeoutError",
+    "crypto",
+    "delete",
+    "diagnose",
+    "fingerprint",
+    "get",
+    "get_profile",
+    "head",
+    "inspect",
+    "options",
+    "patch",
+    "post",
+    "pow",
+    "profiles",
+    "put",
+    "request",
+    "ua",
+    "user_agents",
+]

neptls/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

neptls/async_client.py

@@ -0,0 +1,52 @@
+"""Async API backed by the same tested synchronous transport."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from .client import Client
+from .models import Response
+
+
+class AsyncClient:
+    """An asyncio-friendly client with a persistent cookie/profile session."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        self._client = Client(**kwargs)
+
+    async def request(self, method: str, url: str, **kwargs: Any) -> Response:
+        return await asyncio.to_thread(self._client.request, method, url, **kwargs)
+
+    async def get(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("GET", url, **kwargs)
+
+    async def post(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("POST", url, **kwargs)
+
+    async def put(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("PUT", url, **kwargs)
+
+    async def patch(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("PATCH", url, **kwargs)
+
+    async def delete(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("DELETE", url, **kwargs)
+
+    async def head(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("HEAD", url, **kwargs)
+
+    async def options(self, url: str, **kwargs: Any) -> Response:
+        return await self.request("OPTIONS", url, **kwargs)
+
+    def fingerprint(self):
+        return self._client.fingerprint()
+
+    async def close(self) -> None:
+        await asyncio.to_thread(self._client.close)
+
+    async def __aenter__(self) -> "AsyncClient":
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        await self.close()

neptls/cli.py

@@ -0,0 +1,76 @@
+"""Command-line interface for NepTLS."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict
+
+from . import __version__, get, profiles, ua
+from .crypto import blake2, md5, sha256, sha512
+from .diagnostics import inspect
+from .pow import Challenge, solve
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="neptls", description="NepTLS networking research toolkit")
+    sub = parser.add_subparsers(dest="command")
+    for method in ("get", "post", "put", "patch", "delete", "head", "options"):
+        command = sub.add_parser(method, help=f"perform an HTTP {method.upper()} request")
+        command.add_argument("url")
+        command.add_argument("--json", dest="json_body", help="JSON request body")
+        command.add_argument("--header", action="append", default=[], metavar="NAME:VALUE")
+    for name in ("inspect", "diagnose"):
+        command = sub.add_parser(name, help="inspect DNS, TCP, TLS, and HTTP")
+        command.add_argument("url")
+        command.add_argument("--json", action="store_true", dest="as_json")
+    sub.add_parser("version")
+    profile_command = sub.add_parser("profile", help="list browser profiles")
+    profile_command.add_argument("name", nargs="?")
+    ua_command = sub.add_parser("ua", help="query the user-agent catalog")
+    ua_command.add_argument("kind", nargs="?", default="random", choices=("random", "chrome", "firefox", "mobile"))
+    hash_command = sub.add_parser("hash", help="hash text")
+    hash_command.add_argument("value")
+    hash_command.add_argument("--algorithm", choices=("sha256", "sha512", "blake2", "md5"), default="sha256")
+    pow_command = sub.add_parser("pow", help="solve a generic proof-of-work challenge")
+    pow_command.add_argument("payload")
+    pow_command.add_argument("--difficulty", type=int, default=4)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    if args.command == "version":
+        print(__version__)
+        return 0
+    if args.command in {"get", "post", "put", "patch", "delete", "head", "options"}:
+        headers = dict(item.split(":", 1) for item in args.header if ":" in item)
+        body = json.loads(args.json_body) if args.json_body else None
+        response = getattr(__import__("neptls"), args.command)(args.url, headers=headers, json=body)
+        print(response.text)
+        return 0 if response.ok else 1
+    if args.command in {"inspect", "diagnose"}:
+        print(json.dumps(inspect(args.url), indent=2, default=str))
+        return 0
+    if args.command == "profile":
+        value = profiles.PROFILES if not args.name else profiles.get_profile(args.name).to_dict()
+        print(json.dumps(value, indent=2, default=str))
+        return 0
+    if args.command == "ua":
+        print(getattr(ua, args.kind)())
+        return 0
+    if args.command == "hash":
+        function = {"sha256": sha256, "sha512": sha512, "blake2": blake2, "md5": md5}[args.algorithm]
+        print(function(args.value))
+        return 0
+    if args.command == "pow":
+        result = solve(Challenge(args.payload, difficulty=args.difficulty))
+        print(json.dumps(asdict(result), default=str))
+        return 0
+    _parser().print_help()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

neptls/client.py

@@ -0,0 +1,285 @@
+"""Synchronous requests-like HTTP client built on Python's standard library."""
+
+from __future__ import annotations
+
+import io
+import json as json_module
+import mimetypes
+import random
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Callable, Mapping
+from http.cookiejar import CookieJar
+from pathlib import Path
+from typing import Any
+
+from .exceptions import RequestError, TimeoutError
+from .models import RequestTiming, Response
+from .profiles import BrowserProfile, get_profile
+from .tls import TLSConfig, TLSFingerprint
+
+AuthHook = Callable[[urllib.request.Request], None]
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *_: Any) -> None:
+        return None
+
+
+def _multipart(fields: Mapping[str, Any], files: Mapping[str, Any]) -> tuple[bytes, str]:
+    boundary = f"----neptls-{random.randrange(10**12):012d}"
+    buffer = io.BytesIO()
+    for name, value in fields.items():
+        buffer.write(f"--{boundary}\r\n".encode())
+        buffer.write(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode())
+        buffer.write(str(value).encode())
+        buffer.write(b"\r\n")
+    for name, value in files.items():
+        filename, content, content_type = name, value, None
+        if isinstance(value, tuple):
+            filename, content = value[0], value[1]
+            content_type = value[2] if len(value) > 2 else None
+        if hasattr(content, "read"):
+            content = content.read()
+        elif isinstance(content, (str, Path)) and Path(content).exists():
+            path = Path(content)
+            filename, content = filename or path.name, path.read_bytes()
+        elif isinstance(content, str):
+            content = content.encode()
+        content_type = content_type or mimetypes.guess_type(str(filename))[0] or "application/octet-stream"
+        buffer.write(f"--{boundary}\r\n".encode())
+        buffer.write(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\r\n'.encode())
+        buffer.write(f"Content-Type: {content_type}\r\n\r\n".encode())
+        buffer.write(content)
+        buffer.write(b"\r\n")
+    buffer.write(f"--{boundary}--\r\n".encode())
+    return buffer.getvalue(), f"multipart/form-data; boundary={boundary}"
+
+
+class Client:
+    """Reusable HTTP client with cookies, connection-aware opener, and retries."""
+
+    def __init__(
+        self,
+        *,
+        profile: str | BrowserProfile | None = None,
+        proxy: str | None = None,
+        headers: Mapping[str, str] | None = None,
+        cookies: Mapping[str, str] | None = None,
+        timeout: float = 30.0,
+        verify: bool = True,
+        follow_redirects: bool = True,
+        retries: int = 0,
+        backoff_factor: float = 0.2,
+    ) -> None:
+        self.profile = get_profile(profile)
+        self.timeout = timeout
+        self.follow_redirects = follow_redirects
+        self.retries = max(0, retries)
+        self.backoff_factor = max(0.0, backoff_factor)
+        self.cookies = CookieJar()
+        self._headers = dict(self.profile.headers if self.profile else {})
+        if self.profile:
+            self._headers.setdefault("User-Agent", self.profile.user_agent)
+        if headers:
+            self._headers.update(headers)
+        if cookies:
+            for name, value in cookies.items():
+                self.cookies.set_cookie(
+                    __import__("http.cookiejar", fromlist=["Cookie"]).Cookie(
+                        version=0,
+                        name=name,
+                        value=value,
+                        port=None,
+                        port_specified=False,
+                        domain="",
+                        domain_specified=False,
+                        domain_initial_dot=False,
+                        path="/",
+                        path_specified=True,
+                        secure=False,
+                        expires=None,
+                        discard=True,
+                        comment=None,
+                        comment_url=None,
+                        rest={},
+                        rfc2109=False,
+                    )
+                )
+        if self.profile:
+            tls_config = self.profile.tls
+        else:
+            tls_config = TLSConfig(verify=verify)
+        if not verify:
+            tls_config = TLSConfig(**{**tls_config.to_dict(), "verify": False})
+        handlers: list[Any] = [
+            urllib.request.HTTPCookieProcessor(self.cookies),
+            urllib.request.HTTPSHandler(context=tls_config.ssl_context()),
+        ]
+        if proxy:
+            handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
+        if not follow_redirects:
+            handlers.append(_NoRedirect())
+        self._opener = urllib.request.build_opener(*handlers)
+
+    def _url(self, url: str, params: Mapping[str, Any] | None) -> str:
+        if not params:
+            return url
+        parsed = urllib.parse.urlsplit(url)
+        query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
+        query.extend((key, str(value)) for key, value in params.items())
+        return urllib.parse.urlunsplit(parsed._replace(query=urllib.parse.urlencode(query)))
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        params: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+        data: bytes | str | None = None,
+        json: Any = None,
+        form: Mapping[str, Any] | None = None,
+        files: Mapping[str, Any] | None = None,
+        timeout: float | None = None,
+        stream: bool = False,
+        auth: AuthHook | None = None,
+    ) -> Response:
+        url = self._url(url, params)
+        request_headers = dict(self._headers)
+        if headers:
+            request_headers.update(headers)
+        body: bytes | None = None
+        if json is not None:
+            body = json_module.dumps(json).encode()
+            request_headers.setdefault("Content-Type", "application/json")
+        elif files:
+            body, content_type = _multipart(form or {}, files)
+            request_headers.setdefault("Content-Type", content_type)
+        elif form is not None:
+            body = urllib.parse.urlencode(form).encode()
+            request_headers.setdefault("Content-Type", "application/x-www-form-urlencoded")
+        elif data is not None:
+            body = data.encode() if isinstance(data, str) else data
+        request = urllib.request.Request(url, data=body, headers=request_headers, method=method.upper())
+        if auth:
+            auth(request)
+        last_error: Exception | None = None
+        retry_statuses = {429, 500, 502, 503, 504}
+        for attempt in range(self.retries + 1):
+            started = time.perf_counter()
+            try:
+                raw = self._opener.open(request, timeout=timeout or self.timeout)
+                response = Response(
+                    status_code=raw.status,
+                    reason=raw.reason or "",
+                    url=raw.geturl(),
+                    headers=dict(raw.headers.items()),
+                    raw=raw if stream else None,
+                    content=b"" if stream else raw.read(),
+                    request_method=method.upper(),
+                    timing=RequestTiming(started, time.perf_counter(), first_byte=time.perf_counter()),
+                )
+                if not stream:
+                    raw.close()
+                if response.status_code in retry_statuses and attempt < self.retries:
+                    time.sleep(self.backoff_factor * (2**attempt))
+                    continue
+                return response
+            except urllib.error.HTTPError as exc:
+                response = Response(
+                    status_code=exc.code,
+                    reason=exc.reason or "",
+                    url=exc.geturl(),
+                    headers=dict(exc.headers.items()) if exc.headers else {},
+                    raw=exc if stream else None,
+                    content=b"" if stream else exc.read(),
+                    request_method=method.upper(),
+                    timing=RequestTiming(started, time.perf_counter(), first_byte=time.perf_counter()),
+                )
+                if not stream:
+                    exc.close()
+                if exc.code in retry_statuses and attempt < self.retries:
+                    time.sleep(self.backoff_factor * (2**attempt))
+                    continue
+                return response
+            except TimeoutError as exc:
+                last_error = exc
+                break
+            except (urllib.error.URLError, OSError) as exc:
+                last_error = exc
+                if attempt < self.retries:
+                    time.sleep(self.backoff_factor * (2**attempt))
+                    continue
+                break
+        if isinstance(last_error, TimeoutError):
+            raise TimeoutError(f"request timed out: {url}") from last_error
+        raise RequestError(f"request failed: {url}: {last_error}") from last_error
+
+    def get(self, url: str, **kwargs: Any) -> Response:
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs: Any) -> Response:
+        return self.request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs: Any) -> Response:
+        return self.request("PUT", url, **kwargs)
+
+    def patch(self, url: str, **kwargs: Any) -> Response:
+        return self.request("PATCH", url, **kwargs)
+
+    def delete(self, url: str, **kwargs: Any) -> Response:
+        return self.request("DELETE", url, **kwargs)
+
+    def head(self, url: str, **kwargs: Any) -> Response:
+        return self.request("HEAD", url, **kwargs)
+
+    def options(self, url: str, **kwargs: Any) -> Response:
+        return self.request("OPTIONS", url, **kwargs)
+
+    def fingerprint(self) -> TLSFingerprint:
+        return TLSFingerprint.from_config(self.profile.tls if self.profile else TLSConfig())
+
+    def close(self) -> None:
+        pass
+
+    def __enter__(self) -> "Client":
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+
+def request(method: str, url: str, **kwargs: Any) -> Response:
+    with Client() as client:
+        return client.request(method, url, **kwargs)
+
+
+def get(url: str, **kwargs: Any) -> Response:
+    return request("GET", url, **kwargs)
+
+
+def post(url: str, **kwargs: Any) -> Response:
+    return request("POST", url, **kwargs)
+
+
+def put(url: str, **kwargs: Any) -> Response:
+    return request("PUT", url, **kwargs)
+
+
+def patch(url: str, **kwargs: Any) -> Response:
+    return request("PATCH", url, **kwargs)
+
+
+def delete(url: str, **kwargs: Any) -> Response:
+    return request("DELETE", url, **kwargs)
+
+
+def head(url: str, **kwargs: Any) -> Response:
+    return request("HEAD", url, **kwargs)
+
+
+def options(url: str, **kwargs: Any) -> Response:
+    return request("OPTIONS", url, **kwargs)

neptls/crypto.py

@@ -0,0 +1,62 @@
+"""Small, standard-library protocol encoding and hashing helpers."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import quote, unquote
+
+
+def _bytes(value: str | bytes | bytearray) -> bytes:
+    return value.encode() if isinstance(value, str) else bytes(value)
+
+
+def sha256(value: str | bytes | bytearray) -> str:
+    return hashlib.sha256(_bytes(value)).hexdigest()
+
+
+def sha512(value: str | bytes | bytearray) -> str:
+    return hashlib.sha512(_bytes(value)).hexdigest()
+
+
+def blake2(value: str | bytes | bytearray, *, digest_size: int = 32) -> str:
+    return hashlib.blake2b(_bytes(value), digest_size=digest_size).hexdigest()
+
+
+def md5(value: str | bytes | bytearray) -> str:
+    return hashlib.md5(_bytes(value), usedforsecurity=False).hexdigest()
+
+
+def b64encode(value: str | bytes | bytearray) -> str:
+    return base64.b64encode(_bytes(value)).decode("ascii")
+
+
+def b64decode(value: str | bytes) -> bytes:
+    return base64.b64decode(_bytes(value))
+
+
+def hex_encode(value: str | bytes | bytearray) -> str:
+    return _bytes(value).hex()
+
+
+def hex_decode(value: str) -> bytes:
+    return bytes.fromhex(value)
+
+
+def url_encode(value: str) -> str:
+    return quote(value, safe="")
+
+
+def url_decode(value: str) -> str:
+    return unquote(value)
+
+
+def json_encode(value: Mapping[str, Any] | list[Any]) -> str:
+    return json.dumps(value, separators=(",", ":"), sort_keys=True)
+
+
+def json_decode(value: str | bytes) -> Any:
+    return json.loads(value)

neptls/diagnostics.py

@@ -0,0 +1,59 @@
+"""Read-only DNS, TCP, TLS, and HTTP diagnostics."""
+
+from __future__ import annotations
+
+import socket
+import ssl
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+from .client import Client
+
+
+def inspect(url: str, *, timeout: float = 10.0) -> dict[str, Any]:
+    parsed = urlparse(url)
+    if not parsed.hostname:
+        raise ValueError("URL must include a hostname")
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    result: dict[str, Any] = {
+        "url": url,
+        "hostname": parsed.hostname,
+        "port": port,
+        "scheme": parsed.scheme,
+        "dns": {},
+        "tcp": {},
+        "tls": {},
+        "http": {},
+    }
+    dns_started = time.perf_counter()
+    addresses = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
+    result["dns"] = {
+        "duration": time.perf_counter() - dns_started,
+        "addresses": sorted({item[4][0] for item in addresses}),
+    }
+    tcp_started = time.perf_counter()
+    with socket.create_connection((parsed.hostname, port), timeout=timeout) as raw:
+        result["tcp"] = {"duration": time.perf_counter() - tcp_started, "connected": True}
+        if parsed.scheme == "https":
+            context = ssl.create_default_context()
+            tls_started = time.perf_counter()
+            with context.wrap_socket(raw, server_hostname=parsed.hostname) as secure:
+                result["tls"] = {
+                    "duration": time.perf_counter() - tls_started,
+                    "version": secure.version(),
+                    "cipher": secure.cipher()[0] if secure.cipher() else None,
+                    "alpn": secure.selected_alpn_protocol(),
+                    "peer_certificate": secure.getpeercert(),
+                }
+    response = Client(timeout=timeout, retries=0).head(url)
+    result["http"] = {
+        "status_code": response.status_code,
+        "headers": response.headers,
+        "http_version": response.headers.get("x-http-version", "unknown"),
+        "duration": response.timing.total if response.timing else None,
+    }
+    return result
+
+
+diagnose = inspect

neptls/exceptions.py

@@ -0,0 +1,29 @@
+"""NepTLS exception hierarchy."""
+
+
+class NepTLSError(Exception):
+    """Base class for all NepTLS errors."""
+
+
+class RequestError(NepTLSError):
+    """A network request could not be completed."""
+
+
+class TimeoutError(RequestError):
+    """A request exceeded its configured timeout."""
+
+
+class HTTPStatusError(RequestError):
+    """The server returned an unsuccessful HTTP status."""
+
+    def __init__(self, message: str, response: object) -> None:
+        super().__init__(message)
+        self.response = response
+
+
+class InvalidProfileError(NepTLSError):
+    """A browser or TLS profile is invalid."""
+
+
+class PowError(NepTLSError):
+    """A proof-of-work challenge is invalid or unsolved."""