fixturify 0.1.9__py3-none-any.whl

fixturify/http_d/_recorder.py

@@ -0,0 +1,1350 @@
+"""Recording logic for capturing HTTP interactions."""
+
+import base64
+import hashlib
+import json
+import mimetypes
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+from urllib.parse import parse_qs, urlparse
+
+from fixturify._utils import ENCODING
+
+from fixturify.http_d._models import HttpMapping, HttpMappings, HttpRequest, HttpResponse
+
+
+# Content types that should be stored as separate binary files
+BINARY_CONTENT_TYPES = (
+    "image/",
+    "audio/",
+    "video/",
+    "application/octet-stream",
+    "application/pdf",
+    "application/zip",
+    "application/gzip",
+    "application/x-tar",
+    "application/x-rar",
+    "application/x-7z-compressed",
+    "application/vnd.ms-excel",
+    "application/vnd.openxmlformats",
+    "application/msword",
+    "font/",
+)
+
+
+def _normalize_content_type(content_type: Union[str, bytes, None]) -> str:
+    """Normalize content type to string."""
+    if content_type is None:
+        return ""
+    if isinstance(content_type, bytes):
+        return content_type.decode("utf-8", errors="ignore")
+    return content_type
+
+
+def _is_text_content_type(content_type: Union[str, bytes]) -> bool:
+    """Check if content type indicates text content."""
+    text_types = (
+        "text/",
+        "application/json",
+        "application/xml",
+        "application/javascript",
+        "application/x-www-form-urlencoded",
+    )
+    content_str = _normalize_content_type(content_type)
+    return any(t in content_str.lower() for t in text_types)
+
+
+def _is_json_content_type(content_type: Union[str, bytes]) -> bool:
+    """Check if content type indicates JSON content."""
+    content_str = _normalize_content_type(content_type)
+    return "application/json" in content_str.lower()
+
+
+def _is_binary_content_type(content_type: Union[str, bytes]) -> bool:
+    """Check if content type indicates binary content that should be stored as file."""
+    content_str = _normalize_content_type(content_type)
+    content_lower = content_str.lower()
+    return any(t in content_lower for t in BINARY_CONTENT_TYPES)
+
+
+def _try_parse_json(content: str) -> Any:
+    """Try to parse string as JSON, return parsed or original string."""
+    try:
+        return json.loads(content)
+    except (json.JSONDecodeError, TypeError):
+        return content
+
+
+def _get_file_extension(content_type: Union[str, bytes]) -> str:
+    """Get appropriate file extension for a content type."""
+    # Common mappings
+    extension_map = {
+        "application/json": ".json",
+        "application/pdf": ".pdf",
+        "application/zip": ".zip",
+        "application/gzip": ".gz",
+        "application/xml": ".xml",
+        "text/html": ".html",
+        "text/plain": ".txt",
+        "text/css": ".css",
+        "text/javascript": ".js",
+        "image/jpeg": ".jpg",
+        "image/png": ".png",
+        "image/gif": ".gif",
+        "image/webp": ".webp",
+        "image/svg+xml": ".svg",
+        "audio/mpeg": ".mp3",
+        "audio/wav": ".wav",
+        "video/mp4": ".mp4",
+        "video/webm": ".webm",
+    }
+
+    # Normalize content_type to string
+    content_str = _normalize_content_type(content_type)
+
+    # Check our map first
+    content_lower = content_str.lower().split(";")[0].strip()
+    if content_lower in extension_map:
+        return extension_map[content_lower]
+
+    # Fall back to mimetypes
+    ext = mimetypes.guess_extension(content_lower)
+    if ext:
+        return ext
+
+    # Default to .bin for unknown binary
+    return ".bin"
+
+
+def _serialize_body(
+    body: Optional[Union[bytes, str]],
+    content_type: Union[str, bytes] = "",
+) -> Tuple[Any, Optional[str], bool]:
+    """
+    Serialize body for storage.
+
+    For JSON content: returns parsed JSON object (not stringified)
+    For text content: returns string
+    For binary content: returns raw bytes and flag to save as file
+
+    Args:
+        body: Request or response body (bytes, string, or BytesIO)
+        content_type: Content-Type header value (str or bytes)
+
+    Returns:
+        Tuple of (body_value, encoding, is_binary_file)
+        - For text/JSON: (value, None, False)
+        - For binary: (bytes, None, True)
+    """
+    if body is None:
+        return None, None, False
+
+    # Handle BytesIO and other file-like objects
+    if hasattr(body, "read"):
+        file_obj = body
+        body = file_obj.read()
+        # Reset position if possible for re-reading
+        if hasattr(file_obj, "seek"):
+            file_obj.seek(0)
+
+    # Normalize content_type to string
+    content_type = _normalize_content_type(content_type)
+
+    # Check if this should be stored as a binary file
+    if _is_binary_content_type(content_type):
+        if isinstance(body, bytes):
+            return body, None, True
+        elif isinstance(body, str):
+            return body.encode("utf-8"), None, True
+        else:
+            # Unknown type - try to convert
+            return str(body).encode("utf-8"), None, True
+
+    # Convert bytes to string if needed
+    if isinstance(body, bytes):
+        if _is_text_content_type(content_type) or not content_type:
+            try:
+                body_str = body.decode("utf-8")
+            except UnicodeDecodeError:
+                # Binary content - return as bytes for file storage
+                return body, None, True
+        else:
+            # Unknown content type with bytes - try text first
+            try:
+                body_str = body.decode("utf-8")
+            except UnicodeDecodeError:
+                return body, None, True
+    else:
+        body_str = body
+
+    # Empty body
+    if not body_str:
+        return None, None, False
+
+    # For JSON content, parse and store as native JSON
+    if _is_json_content_type(content_type):
+        return _try_parse_json(body_str), None, False
+
+    # Return as string for non-JSON text content
+    return body_str, None, False
+
+
+class HttpRecorder:
+    """
+    Records HTTP interactions for later playback.
+
+    Captures request/response pairs and saves them to a JSON file
+    in WireMock-compatible format. Binary responses are stored as
+    separate files in a __files subdirectory.
+    """
+
+    # Directory name for binary response files (WireMock convention)
+    FILES_DIR = "__files"
+
+    def __init__(
+        self,
+        file_path: Path,
+        exclude_request_headers: Optional[set] = None,
+        exclude_response_headers: Optional[set] = None,
+        redact_request_body: Optional[list[str]] = None,
+        redact_response_body: Optional[list[str]] = None,
+    ):
+        """
+        Initialize the recorder.
+
+        Args:
+            file_path: Path where recordings will be saved.
+            exclude_request_headers: Set of header names (lowercase) to exclude
+                                    from request recordings.
+            exclude_response_headers: Set of header names (lowercase) to exclude
+                                     from response recordings.
+        """
+        self.file_path = file_path
+        self.mappings: HttpMappings = HttpMappings()
+        # Track binary files to save: list of (filename, bytes)
+        self._binary_files: List[Tuple[str, bytes]] = []
+        # Counter for generating unique filenames
+        self._file_counter = 0
+        # Headers to exclude from recordings
+        self._exclude_request_headers = exclude_request_headers or set()
+        self._exclude_response_headers = exclude_response_headers or set()
+        self._redact_request_body = {f for f in (redact_request_body or []) if f}
+        self._redact_response_body = {f for f in (redact_response_body or []) if f}
+
+    def _redact_json_fields(self, value: Any) -> Any:
+        """Redact configured JSON fields in dict/list structures."""
+        if not self._redact_request_body:
+            return value
+        if isinstance(value, dict):
+            redacted = {}
+            for key, val in value.items():
+                if key in self._redact_request_body:
+                    redacted[key] = "*******"
+                else:
+                    redacted[key] = self._redact_json_fields(val)
+            return redacted
+        if isinstance(value, list):
+            return [self._redact_json_fields(item) for item in value]
+        return value
+
+    def _redact_json_fields_response(self, value: Any) -> Any:
+        """Redact configured JSON fields in dict/list structures (response)."""
+        if not self._redact_response_body:
+            return value
+        if isinstance(value, dict):
+            redacted = {}
+            for key, val in value.items():
+                if key in self._redact_response_body:
+                    redacted[key] = "*******"
+                else:
+                    redacted[key] = self._redact_json_fields_response(val)
+            return redacted
+        if isinstance(value, list):
+            return [self._redact_json_fields_response(item) for item in value]
+        return value
+
+    def record(
+        self,
+        request: HttpRequest,
+        response: HttpResponse,
+    ) -> None:
+        """
+        Record a request/response pair.
+
+        Args:
+            request: The HTTP request.
+            response: The HTTP response.
+        """
+        mapping = HttpMapping(request=request, response=response)
+        self.mappings.add(mapping)
+
+    def save(self) -> None:
+        """
+        Save all recorded mappings to the JSON file.
+
+        Creates parent directories if they don't exist.
+        Binary response bodies are saved to a __files subdirectory.
+        """
+        # Ensure parent directory exists
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Convert to dict with native JSON bodies (and save binary files)
+        data = self._to_dict_with_json_bodies()
+
+        # Write JSON file
+        with open(self.file_path, "w", encoding=ENCODING) as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+
+        # Save binary files
+        if self._binary_files:
+            files_dir = self.file_path.parent / self.FILES_DIR
+            files_dir.mkdir(exist_ok=True)
+
+            for filename, content in self._binary_files:
+                file_path = files_dir / filename
+                with open(file_path, "wb") as f:
+                    f.write(content)
+
+    def _generate_filename(self, content_type: str, content: bytes) -> str:
+        """
+        Generate a unique filename for binary content.
+
+        Uses a hash of the content to enable deduplication.
+
+        Args:
+            content_type: Content-Type header
+            content: Binary content
+
+        Returns:
+            Filename like "response_0_a1b2c3d4.png"
+        """
+        self._file_counter += 1
+        ext = _get_file_extension(content_type)
+        # Use content hash for deduplication and uniqueness
+        content_hash = hashlib.sha256(content).hexdigest()[:8]
+        return f"response_{self._file_counter}_{content_hash}{ext}"
+
+    def _filter_headers(
+        self,
+        headers: Dict[str, str],
+        exclude_set: set,
+    ) -> Dict[str, str]:
+        """
+        Filter headers, removing excluded ones.
+
+        Args:
+            headers: Original headers dict.
+            exclude_set: Set of lowercase header names to exclude.
+
+        Returns:
+            Filtered headers dict.
+        """
+        if not exclude_set:
+            return headers
+        return {k: v for k, v in headers.items() if k.lower() not in exclude_set}
+
+    def _to_dict_with_json_bodies(self) -> Dict[str, Any]:
+        """
+        Convert mappings to dict, storing bodies as native JSON.
+
+        Binary response bodies are saved to files and referenced via bodyFileName.
+        Excluded headers are filtered out from the recording.
+
+        Returns:
+            Dict ready for JSON serialization.
+        """
+        mappings_list = []
+        self._binary_files.clear()
+        self._file_counter = 0
+
+        for mapping in self.mappings:
+            # Filter headers
+            req_headers = self._filter_headers(
+                mapping.request.headers,
+                self._exclude_request_headers,
+            )
+            resp_headers = self._filter_headers(
+                mapping.response.headers,
+                self._exclude_response_headers,
+            )
+
+            # Get content types for body serialization
+            req_content_type = req_headers.get(
+                "Content-Type",
+                req_headers.get("content-type", ""),
+            )
+            resp_content_type = resp_headers.get(
+                "Content-Type",
+                resp_headers.get("content-type", ""),
+            )
+
+            # Serialize request body
+            req_body = mapping.request.body
+            if isinstance(req_body, str) and _is_json_content_type(req_content_type):
+                req_body = _try_parse_json(req_body)
+
+            # Build request dict
+            if isinstance(req_body, (dict, list)):
+                req_body = self._redact_json_fields(req_body)
+            request_dict: Dict[str, Any] = {
+                "method": mapping.request.method.upper(),
+                "url": mapping.request.url,
+                "headers": req_headers,
+                "body": req_body,
+            }
+            if mapping.request.bodyEncoding:
+                request_dict["bodyEncoding"] = mapping.request.bodyEncoding
+
+            # Build response dict
+            response_dict: Dict[str, Any] = {
+                "status": mapping.response.status,
+                "headers": resp_headers,
+            }
+
+            # Check if response has a binary file reference
+            if mapping.response.bodyFileName:
+                response_dict["bodyFileName"] = mapping.response.bodyFileName
+            elif mapping.response._body_bytes is not None:
+                # Binary content that needs to be saved as file
+                filename = self._generate_filename(
+                    resp_content_type,
+                    mapping.response._body_bytes,
+                )
+                self._binary_files.append((filename, mapping.response._body_bytes))
+                response_dict["bodyFileName"] = filename
+            elif _is_binary_content_type(resp_content_type) and mapping.response.body:
+                # Response body is base64-encoded binary - decode and save as file
+                try:
+                    content = base64.b64decode(mapping.response.body)
+                    filename = self._generate_filename(resp_content_type, content)
+                    self._binary_files.append((filename, content))
+                    response_dict["bodyFileName"] = filename
+                except Exception:
+                    # Fall back to inline body
+                    resp_body = mapping.response.body
+                    if _is_json_content_type(resp_content_type):
+                        resp_body = _try_parse_json(resp_body)
+                    response_dict["body"] = resp_body
+            else:
+                # Text/JSON content - store inline
+                resp_body = mapping.response.body
+                if isinstance(resp_body, str) and _is_json_content_type(
+                    resp_content_type
+                ):
+                    resp_body = _try_parse_json(resp_body)
+                if isinstance(resp_body, (dict, list)):
+                    resp_body = self._redact_json_fields_response(resp_body)
+                response_dict["body"] = resp_body if resp_body else None
+
+            if mapping.response.bodyEncoding and "bodyFileName" not in response_dict:
+                response_dict["bodyEncoding"] = mapping.response.bodyEncoding
+
+            mappings_list.append(
+                {
+                    "request": request_dict,
+                    "response": response_dict,
+                }
+            )
+
+        return {"mappings": mappings_list}
+
+    def __len__(self) -> int:
+        """Return number of recorded mappings."""
+        return len(self.mappings)
+
+
+def create_request_from_httpx(
+    request: Any,  # httpx.Request
+) -> HttpRequest:
+    """
+    Create HttpRequest from an httpx Request object.
+
+    Args:
+        request: httpx.Request object
+
+    Returns:
+        HttpRequest model
+    """
+    # Get URL and extract query parameters
+    url_str = str(request.url)
+    parsed = urlparse(url_str)
+
+    # Extract query parameters
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    # Get headers as dict
+    headers = dict(request.headers)
+
+    # Get body
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+    body_bytes = request.content if hasattr(request, "content") else None
+    body, body_encoding, _ = _serialize_body(body_bytes, content_type)
+
+    # Convert body back to string for storage in model (will be parsed when saving)
+    body_str = None
+    if body is not None:
+        if body_encoding == "base64":
+            body_str = body
+        elif isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpRequest(
+        method=request.method,
+        url=url_str,
+        headers=headers,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_httpx(
+    response: Any,  # httpx.Response
+) -> HttpResponse:
+    """
+    Create HttpResponse from an httpx Response object.
+
+    Args:
+        response: httpx.Response object
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.headers)
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    # Get response body
+    body_bytes = response.content
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        # Binary content - store bytes for file saving
+        resp = HttpResponse(
+            status=response.status_code,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    # Convert body to string for storage in model
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status_code,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_request_from_requests(
+    prepared_request: Any,  # requests.PreparedRequest
+) -> HttpRequest:
+    """
+    Create HttpRequest from a requests PreparedRequest object.
+
+    Args:
+        prepared_request: requests.PreparedRequest object
+
+    Returns:
+        HttpRequest model
+    """
+    url_str = prepared_request.url or ""
+    parsed = urlparse(url_str)
+
+    # Extract query parameters
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    # Get headers as dict
+    headers = dict(prepared_request.headers) if prepared_request.headers else {}
+
+    # Get body
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+    body_raw = prepared_request.body
+    body, body_encoding, _ = _serialize_body(body_raw, content_type)
+
+    # Convert body back to string for storage
+    body_str = None
+    if body is not None:
+        if body_encoding == "base64":
+            body_str = body
+        elif isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpRequest(
+        method=prepared_request.method or "GET",
+        url=url_str,
+        headers=headers,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_requests(
+    response: Any,  # requests.Response
+) -> HttpResponse:
+    """
+    Create HttpResponse from a requests Response object.
+
+    Args:
+        response: requests.Response object
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.headers) if response.headers else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    # Get response body
+    body_bytes = response.content
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        # Binary content - store bytes for file saving
+        resp = HttpResponse(
+            status=response.status_code,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    # Convert body to string for storage
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status_code,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# urllib3 functions
+# ============================================================================
+
+
+def create_request_from_urllib3(
+    method: str,
+    url: str,
+    body: Any = None,
+    headers: Any = None,
+) -> HttpRequest:
+    """
+    Create HttpRequest from urllib3 request parameters.
+
+    Args:
+        method: HTTP method
+        url: Full URL string
+        body: Request body
+        headers: Request headers
+
+    Returns:
+        HttpRequest model
+    """
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    # Normalize headers - may contain bytes values from botocore
+    headers_dict: Dict[str, str] = {}
+    if headers:
+        for k, v in (headers.items() if hasattr(headers, "items") else headers):
+            key = k.decode("utf-8") if isinstance(k, bytes) else str(k)
+            val = v.decode("utf-8") if isinstance(v, bytes) else str(v)
+            headers_dict[key] = val
+
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body_value, body_encoding, _ = _serialize_body(body, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=method,
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_urllib3(
+    response: Any,  # urllib3.HTTPResponse
+) -> HttpResponse:
+    """
+    Create HttpResponse from a urllib3 HTTPResponse object.
+
+    Args:
+        response: urllib3.HTTPResponse object
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.headers) if response.headers else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    body_bytes = response.data
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.status,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# http.client functions
+# ============================================================================
+
+
+def create_request_from_http_client(
+    method: str,
+    url: str,
+    body: Any = None,
+    headers: Dict[str, str] = None,
+) -> HttpRequest:
+    """
+    Create HttpRequest from http.client request parameters.
+
+    Args:
+        method: HTTP method
+        url: Full URL string
+        body: Request body
+        headers: Request headers dict
+
+    Returns:
+        HttpRequest model
+    """
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    headers_dict = headers or {}
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body_value, body_encoding, _ = _serialize_body(body, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=method,
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_http_client(
+    response: Any,  # http.client.HTTPResponse
+) -> HttpResponse:
+    """
+    Create HttpResponse from an http.client HTTPResponse object.
+
+    Args:
+        response: http.client.HTTPResponse object
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.getheaders()) if hasattr(response, "getheaders") else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    body_bytes = response.read()
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.status,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# aiohttp functions
+# ============================================================================
+
+
+def create_request_from_aiohttp(
+    method: str,
+    url: str,
+    data: Any = None,
+    headers: Any = None,
+) -> HttpRequest:
+    """
+    Create HttpRequest from aiohttp request parameters.
+
+    Args:
+        method: HTTP method
+        url: URL string
+        data: Request body data
+        headers: Request headers
+
+    Returns:
+        HttpRequest model
+    """
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    headers_dict = dict(headers) if headers else {}
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body_value, body_encoding, _ = _serialize_body(data, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=method.upper(),
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_aiohttp(
+    response: Any,  # aiohttp.ClientResponse
+    body_bytes: bytes,
+) -> HttpResponse:
+    """
+    Create HttpResponse from an aiohttp ClientResponse object.
+
+    Args:
+        response: aiohttp.ClientResponse object
+        body_bytes: Already-read response body bytes
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.headers) if response.headers else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.status,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# httplib2 functions
+# ============================================================================
+
+
+def create_request_from_httplib2(
+    uri: str,
+    method: str = "GET",
+    body: Any = None,
+    headers: Any = None,
+) -> HttpRequest:
+    """
+    Create HttpRequest from httplib2 request parameters.
+
+    Args:
+        uri: Request URI
+        method: HTTP method
+        body: Request body
+        headers: Request headers
+
+    Returns:
+        HttpRequest model
+    """
+    parsed = urlparse(uri)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    headers_dict = dict(headers) if headers else {}
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body_value, body_encoding, _ = _serialize_body(body, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=method.upper(),
+        url=uri,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_httplib2(
+    response: Any,  # httplib2.Response
+    content: bytes,
+) -> HttpResponse:
+    """
+    Create HttpResponse from an httplib2 Response object.
+
+    Args:
+        response: httplib2.Response object
+        content: Response body bytes
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response) if response else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    body, body_encoding, is_binary = _serialize_body(content, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.status,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else content
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# tornado functions
+# ============================================================================
+
+
+def create_request_from_tornado(
+    request: Any,  # tornado.httpclient.HTTPRequest
+) -> HttpRequest:
+    """
+    Create HttpRequest from a tornado HTTPRequest object.
+
+    Args:
+        request: tornado.httpclient.HTTPRequest object
+
+    Returns:
+        HttpRequest model
+    """
+    url = request.url
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    headers_dict = dict(request.headers) if request.headers else {}
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body = request.body
+    body_value, body_encoding, _ = _serialize_body(body, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=request.method or "GET",
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_tornado(
+    response: Any,  # tornado.httpclient.HTTPResponse
+) -> HttpResponse:
+    """
+    Create HttpResponse from a tornado HTTPResponse object.
+
+    Args:
+        response: tornado.httpclient.HTTPResponse object
+
+    Returns:
+        HttpResponse model
+    """
+    headers = dict(response.headers) if response.headers else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    body_bytes = response.body or b""
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.code,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.code,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# botocore functions
+# ============================================================================
+
+
+def create_request_from_botocore(
+    request: Any,  # botocore.awsrequest.AWSRequest
+) -> HttpRequest:
+    """
+    Create HttpRequest from a botocore AWSRequest object.
+
+    Args:
+        request: botocore.awsrequest.AWSRequest object
+
+    Returns:
+        HttpRequest model
+    """
+    url = request.url
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    # Normalize headers - botocore can have bytes values
+    headers_dict: Dict[str, str] = {}
+    if request.headers:
+        for k, v in request.headers.items():
+            key = k.decode("utf-8") if isinstance(k, bytes) else k
+            val = v.decode("utf-8") if isinstance(v, bytes) else v
+            headers_dict[key] = val
+
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    body = request.body
+    body_value, body_encoding, _ = _serialize_body(body, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=request.method or "GET",
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_botocore(
+    response: Any,  # botocore.awsrequest.AWSResponse
+) -> HttpResponse:
+    """
+    Create HttpResponse from a botocore AWSResponse object.
+
+    Args:
+        response: botocore.awsrequest.AWSResponse object
+
+    Returns:
+        HttpResponse model
+    """
+    # AWSResponse has headers as HTTPHeaderDict
+    headers = dict(response.headers) if response.headers else {}
+    content_type = headers.get("content-type", headers.get("Content-Type", ""))
+
+    # AWSResponse has content property for body bytes
+    body_bytes = response.content
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    # AWSResponse uses status_code, not status
+    status = response.status_code
+
+    if is_binary:
+        resp = HttpResponse(
+            status=status,
+            headers=headers,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=status,
+        headers=headers,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+# ============================================================================
+# httpcore functions
+# ============================================================================
+
+
+def create_request_from_httpcore(
+    method: str,
+    url: str,
+    content: Any = None,
+    headers: Any = None,
+) -> HttpRequest:
+    """
+    Create HttpRequest from httpcore request parameters.
+
+    Args:
+        method: HTTP method
+        url: URL string
+        content: Request body content
+        headers: Request headers (list of tuples or bytes pairs)
+
+    Returns:
+        HttpRequest model
+    """
+    parsed = urlparse(url)
+
+    query_params: Dict[str, str] = {}
+    if parsed.query:
+        for key, values in parse_qs(parsed.query).items():
+            query_params[key] = values[0] if values else ""
+
+    # Convert headers from list of tuples to dict
+    headers_dict: Dict[str, str] = {}
+    if headers:
+        for k, v in headers:
+            key = k.decode() if isinstance(k, bytes) else k
+            val = v.decode() if isinstance(v, bytes) else v
+            headers_dict[key] = val
+
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    # Handle content
+    if isinstance(content, bytes):
+        body_data = content
+    elif hasattr(content, "read"):
+        body_data = content.read()
+    else:
+        body_data = content
+
+    body_value, body_encoding, _ = _serialize_body(body_data, content_type)
+
+    body_str = None
+    if body_value is not None:
+        if body_encoding == "base64":
+            body_str = body_value
+        elif isinstance(body_value, str):
+            body_str = body_value
+        else:
+            body_str = json.dumps(body_value, ensure_ascii=False)
+
+    return HttpRequest(
+        method=method.upper(),
+        url=url,
+        headers=headers_dict,
+        queryParameters=query_params,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )
+
+
+def create_response_from_httpcore(
+    response: Any,  # httpcore.Response
+) -> HttpResponse:
+    """
+    Create HttpResponse from an httpcore Response object.
+
+    Args:
+        response: httpcore.Response object
+
+    Returns:
+        HttpResponse model
+    """
+    # Convert headers from list of tuples to dict
+    headers_dict: Dict[str, str] = {}
+    if response.headers:
+        for k, v in response.headers:
+            key = k.decode() if isinstance(k, bytes) else k
+            val = v.decode() if isinstance(v, bytes) else v
+            headers_dict[key] = val
+
+    content_type = headers_dict.get("content-type", headers_dict.get("Content-Type", ""))
+
+    # Get body content
+    body_bytes = response.content
+    body, body_encoding, is_binary = _serialize_body(body_bytes, content_type)
+
+    if is_binary:
+        resp = HttpResponse(
+            status=response.status,
+            headers=headers_dict,
+            body=None,
+        )
+        resp._body_bytes = body if isinstance(body, bytes) else body_bytes
+        return resp
+
+    body_str = ""
+    if body is not None:
+        if isinstance(body, str):
+            body_str = body
+        else:
+            body_str = json.dumps(body, ensure_ascii=False)
+
+    return HttpResponse(
+        status=response.status,
+        headers=headers_dict,
+        body=body_str,
+        bodyEncoding=body_encoding,
+    )