mm-std 0.3.24__py3-none-any.whl → 0.3.26__py3-none-any.whl

mm_std/__init__.py

@@ -19,6 +19,9 @@ from .date import utc_now as utc_now
 from .date import utc_random as utc_random
 from .dict import replace_empty_dict_values as replace_empty_dict_values
 from .env import get_dotenv as get_dotenv
+from .http.http_request import http_request as http_request
+from .http.response import HttpError as HttpError
+from .http.response import HttpResponse as HttpResponse
 from .http_ import CHROME_USER_AGENT as CHROME_USER_AGENT
 from .http_ import FIREFOX_USER_AGENT as FIREFOX_USER_AGENT
 from .http_ import HResponse as HResponse

mm_std/http/http_request.py

@@ -0,0 +1,121 @@
+import aiohttp
+from aiohttp.typedefs import LooseCookies, Query
+from aiohttp_socks import ProxyConnector
+from multidict import CIMultiDictProxy
+
+from mm_std.http.response import HttpError, HttpResponse
+
+
+async def http_request(
+    url: str,
+    *,
+    method: str = "GET",
+    params: Query | None = None,
+    data: dict[str, object] | None = None,
+    json: dict[str, object] | None = None,
+    headers: dict[str, str] | None = None,
+    cookies: LooseCookies | None = None,
+    user_agent: str | None = None,
+    proxy: str | None = None,
+    timeout: float | None = 10.0,
+) -> HttpResponse:
+    """
+    Send an HTTP request and return the response.
+    """
+    timeout_ = aiohttp.ClientTimeout(total=timeout) if timeout else None
+    if user_agent:
+        if not headers:
+            headers = {}
+        headers["user-agent"] = user_agent
+
+    try:
+        if proxy and proxy.startswith("socks"):
+            return await _request_with_socks_proxy(
+                url,
+                method=method,
+                params=params,
+                data=data,
+                json=json,
+                headers=headers,
+                cookies=cookies,
+                proxy=proxy,
+                timeout=timeout_,
+            )
+        return await _request_with_http_or_none_proxy(
+            url,
+            method=method,
+            params=params,
+            data=data,
+            json=json,
+            headers=headers,
+            cookies=cookies,
+            proxy=proxy,
+            timeout=timeout_,
+        )
+    except TimeoutError as err:
+        return HttpResponse(error=HttpError.TIMEOUT, error_message=str(err))
+    except Exception as err:
+        return HttpResponse(error=HttpError.ERROR, error_message=str(err))
+
+
+async def _request_with_http_or_none_proxy(
+    url: str,
+    *,
+    method: str = "GET",
+    params: Query | None = None,
+    data: dict[str, object] | None = None,
+    json: dict[str, object] | None = None,
+    headers: dict[str, str] | None = None,
+    cookies: LooseCookies | None = None,
+    proxy: str | None = None,
+    timeout: aiohttp.ClientTimeout | None,
+) -> HttpResponse:
+    async with aiohttp.request(
+        method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, proxy=proxy, timeout=timeout
+    ) as res:
+        return HttpResponse(
+            status=res.status,
+            error=None,
+            error_message=None,
+            body=(await res.read()).decode(),
+            headers=headers_dict(res.headers),
+        )
+
+
+async def _request_with_socks_proxy(
+    url: str,
+    *,
+    method: str = "GET",
+    proxy: str,
+    params: Query | None = None,
+    data: dict[str, object] | None = None,
+    json: dict[str, object] | None = None,
+    headers: dict[str, str] | None = None,
+    cookies: LooseCookies | None = None,
+    timeout: aiohttp.ClientTimeout | None,
+) -> HttpResponse:
+    connector = ProxyConnector.from_url(proxy)
+    async with (
+        aiohttp.ClientSession(connector=connector) as session,
+        session.request(
+            method, url, params=params, data=data, json=json, headers=headers, cookies=cookies, timeout=timeout
+        ) as res,
+    ):
+        return HttpResponse(
+            status=res.status,
+            error=None,
+            error_message=None,
+            body=(await res.read()).decode(),
+            headers=headers_dict(res.headers),
+        )
+
+
+def headers_dict(headers: CIMultiDictProxy[str]) -> dict[str, str]:
+    result: dict[str, str] = {}
+    for key in headers:
+        values = headers.getall(key)
+        if len(values) == 1:
+            result[key] = values[0]
+        else:
+            result[key] = ", ".join(values)
+    return result

mm_std/http/response.py

@@ -0,0 +1,93 @@
+import enum
+import json
+from typing import Any
+
+import pydash
+from pydantic import GetCoreSchemaHandler
+from pydantic_core import CoreSchema, core_schema
+
+
+@enum.unique
+class HttpError(str, enum.Enum):
+    TIMEOUT = "timeout"
+    PROXY = "proxy"
+    CONNECTION = "connection"
+    ERROR = "error"
+
+
+class HttpResponse:
+    def __init__(
+        self,
+        status: int | None = None,
+        error: HttpError | None = None,
+        error_message: str | None = None,
+        body: str | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> None:
+        self.status = status
+        self.error = error
+        self.error_message = error_message
+        self.body = body
+        self.headers = headers
+
+        self._json_data: Any = None
+        self._json_parsed = False
+        self._json_parsed_error = False
+
+    def _parse_json(self) -> None:
+        if self.body is None:
+            self._json_parsed_error = True
+            return
+        try:
+            self._json_data = None
+            self._json_data = json.loads(self.body)
+            self._json_parsed_error = False
+        except json.JSONDecodeError:
+            self._json_parsed_error = True
+        self._json_parsed = True
+
+    def json(self, path: str | None = None) -> Any:  # noqa: ANN401
+        if not self._json_parsed:
+            self._parse_json()
+        if path:
+            return pydash.get(self._json_data, path, None)
+        return self._json_data
+
+    def dict(self) -> dict[str, object]:
+        return {
+            "status": self.status,
+            "error": self.error,
+            "error_message": self.error_message,
+            "body": self.body,
+            "headers": self.headers,
+        }
+
+    def is_json_parse_error(self) -> bool:
+        if not self._json_parsed:
+            self._parse_json()
+        return self._json_parsed_error
+
+    def __repr__(self) -> str:
+        return f"HttpResponse(status={self.status}, error={self.error}, error_message={self.error_message}, body={self.body}, headers={self.headers})"  # noqa: E501
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: type[Any], handler: GetCoreSchemaHandler) -> CoreSchema:
+        return core_schema.no_info_after_validator_function(
+            cls._validate,
+            core_schema.any_schema(),
+            serialization=core_schema.plain_serializer_function_ser_schema(lambda x: x.dict()),
+        )
+
+    @classmethod
+    def _validate(cls, v: object) -> "HttpResponse":
+        if isinstance(v, cls):
+            return v
+        if isinstance(v, dict):
+            return cls(
+                status=v.get("status"),
+                error=HttpError(v["error"]) if v.get("error") else None,
+                error_message=v.get("error_message"),
+                body=v.get("body"),
+                headers=v.get("headers"),
+            )
+        raise TypeError(f"Cannot parse value as {cls.__name__}: {v}")

mm_std/http_.py

@@ -1,12 +1,13 @@
 import json
 from dataclasses import asdict, dataclass, field
-from typing import Any
-from urllib.parse import urlencode
+from typing import Any, cast
+from urllib.parse import urlencode, urlparse
 
 import aiohttp
 import pydash
 import requests
 from aiohttp_socks import ProxyConnector
+from multidict import CIMultiDictProxy
 from requests.auth import AuthBase
 
 from mm_std.result import Err, Ok, Result
@@ -77,6 +78,9 @@ class HResponse:
     def is_connection_error(self) -> bool:
         return self.error is not None and self.error.startswith("connection:")
 
+    def is_dns_error(self) -> bool:
+        return self.error is not None and self.error.startswith("dns:")
+
     def to_dict(self) -> dict[str, Any]:
         return pydash.omit(asdict(self), "_json_data")
 
@@ -185,52 +189,50 @@ async def hrequest_async(
             request_kwargs["auth"] = aiohttp.BasicAuth(auth[0], auth[1])
 
         if proxy and proxy.startswith("socks"):
-            res = await _aiohttp_socks5(url, method, proxy, request_kwargs, timeout)
-        else:
-            res = await _aiohttp(url, method, request_kwargs, timeout=timeout, proxy=proxy)
+            return await _aiohttp_socks5(url, method, proxy, request_kwargs, timeout)
+        return await _aiohttp(url, method, request_kwargs, timeout=timeout, proxy=proxy)
 
-        return HResponse(code=res.status, body=res.body, headers=res.headers)
     except TimeoutError:
         return HResponse(error="timeout")
-    except (aiohttp.ClientProxyConnectionError, aiohttp.ClientHttpProxyError):
-        return HResponse(error="proxy")
-    except aiohttp.ClientConnectorError as err:
+    except (aiohttp.ClientProxyConnectionError, aiohttp.ClientHttpProxyError, aiohttp.ClientConnectorError) as err:
+        if is_proxy_error(str(err), proxy):
+            return HResponse(error="proxy")
         return HResponse(error=f"connection: {err}")
     except aiohttp.ClientError as err:
         return HResponse(error=f"error: {err}")
     except Exception as err:
-        if "couldn't connect to proxy" in str(err).lower():
-            return HResponse(error="proxy")
         return HResponse(error=f"exception: {err}")
 
 
-@dataclass
-class AioHttpResponse:
-    status: int
-    body: str
-    headers: dict[str, str]
+def is_proxy_error(error_message: str, proxy: str | None) -> bool:
+    if not proxy:
+        return False
+    error_message = error_message.lower()
+    if "proxy" in error_message:
+        return True
+    return bool("cannot connect to" in error_message and cast(str, urlparse(proxy).hostname) in error_message)
 
 
 async def _aiohttp(
     url: str, method: str, request_kwargs: dict[str, object], timeout: float | None = None, proxy: str | None = None
-) -> AioHttpResponse:
+) -> HResponse:
     if proxy:
         request_kwargs["proxy"] = proxy
     client_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
     async with aiohttp.ClientSession(timeout=client_timeout) as session, session.request(method, url, **request_kwargs) as res:  # type: ignore[arg-type]
-        return AioHttpResponse(status=res.status, headers=dict(res.headers), body=await res.text())
+        return HResponse(code=res.status, body=await res.text(), headers=headers_dict(res.headers))
 
 
 async def _aiohttp_socks5(
     url: str, method: str, proxy: str, request_kwargs: dict[str, object], timeout: float | None = None
-) -> AioHttpResponse:
+) -> HResponse:
     connector = ProxyConnector.from_url(proxy)
     client_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
     async with (
         aiohttp.ClientSession(connector=connector, timeout=client_timeout) as session,
         session.request(method, url, **request_kwargs) as res,  # type: ignore[arg-type]
     ):
-        return AioHttpResponse(status=res.status, headers=dict(res.headers), body=await res.text())
+        return HResponse(code=res.status, body=await res.text(), headers=headers_dict(res.headers))
 
 
 def add_query_params_to_url(url: str, params: dict[str, object]) -> str:
@@ -242,3 +244,14 @@ def add_query_params_to_url(url: str, params: dict[str, object]) -> str:
 
 hr = hrequest
 hra = hrequest_async
+
+
+def headers_dict(headers: CIMultiDictProxy[str]) -> dict[str, str]:
+    result: dict[str, str] = {}
+    for key in headers:
+        values = headers.getall(key)
+        if len(values) == 1:
+            result[key] = values[0]
+        else:
+            result[key] = ", ".join(values)
+    return result

{mm_std-0.3.24.dist-info → mm_std-0.3.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mm-std
-Version: 0.3.24
+Version: 0.3.26
 Requires-Python: >=3.12
 Requires-Dist: aiohttp-socks~=0.10.1
 Requires-Dist: aiohttp~=3.11.16

{mm_std-0.3.24.dist-info → mm_std-0.3.26.dist-info}/RECORD

@@ -1,4 +1,4 @@
-mm_std/__init__.py,sha256=bdkBTX7_u4NYtTRdVDHgZUozQlFRBx3TTiMK7Cenup0,3100
+mm_std/__init__.py,sha256=Q0UguFT4b_LdKmxFSI4NQ47mwYRVRTjZ6fgYh8qZY4Q,3266
 mm_std/command.py,sha256=ze286wjUjg0QSTgIu-2WZks53_Vclg69UaYYgPpQvCU,1283
 mm_std/config.py,sha256=4ox4D2CgGR76bvZ2n2vGQOYUDagFnlKEDb87to5zpxE,1871
 mm_std/crypto.py,sha256=jdk0_TCmeU0pPXMyz9xH6kQHSjjZ9GcGClBwQps5vBo,340
@@ -6,7 +6,7 @@ mm_std/date.py,sha256=976eEkSONuNqHQBgSRu8hrtH23tJqztbmHFHLdbP2TY,1879
 mm_std/dict.py,sha256=6GkhJPXD0LiJDxPcYe6jPdEDw-MN7P7mKu6U5XxwYDk,675
 mm_std/env.py,sha256=5zaR9VeIfObN-4yfgxoFeU5IM1GDeZZj9SuYf7t9sOA,125
 mm_std/fs.py,sha256=RwarNRJq3tIMG6LVX_g03hasfYpjYFh_O27oVDt5IPQ,291
-mm_std/http_.py,sha256=A9SUidkmTEFYWXD9y-rX4MrUXtaZFiy84Ul8floYBn8,7700
+mm_std/http_.py,sha256=5_V5Hx1M07xwzf0VVqAsxSFJkvtaAhj9_sFQP_HliQw,8215
 mm_std/json_.py,sha256=Naa6mBE4D0yiQGkPNRrFvndnUH3R7ovw3FeaejWV60o,1196
 mm_std/log.py,sha256=0TkTsAlUTt00gjgukvsvnZRIAGELq0MI6Lv8mKP-Wz4,2887
 mm_std/net.py,sha256=qdRCBIDneip6FaPNe5mx31UtYVmzqam_AoUF7ydEyjA,590
@@ -25,6 +25,9 @@ mm_std/concurrency/async_task_runner.py,sha256=EN7tN2enkVYVgDbhSiAr-_W4o9m9wBXCv
 mm_std/concurrency/sync_decorators.py,sha256=syCQBOmN7qPO55yzgJB2rbkh10CVww376hmyvs6e5tA,1080
 mm_std/concurrency/sync_scheduler.py,sha256=j4tBL_cBI1spr0cZplTA7N2CoYsznuORMeRN8rpR6gY,2407
 mm_std/concurrency/sync_task_runner.py,sha256=s5JPlLYLGQGHIxy4oDS-PN7O9gcy-yPZFoNm8RQwzcw,1780
-mm_std-0.3.24.dist-info/METADATA,sha256=El91GJxSAEQMfJHtBogFAyAsXGcRJ3J9sfCPCNwIehA,415
-mm_std-0.3.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mm_std-0.3.24.dist-info/RECORD,,
+mm_std/http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mm_std/http/http_request.py,sha256=mJak332gv9NvY0JLO3hNrBxroQGS1HlNBzd1yq4Dy24,3694
+mm_std/http/response.py,sha256=vvv5COTjJula9t33mFyrruhrFC4dr_Uy0jDKj6t1JxM,2923
+mm_std-0.3.26.dist-info/METADATA,sha256=GW6fYKKEoBHo1GlukZ2LtY3gII8feRqjIynazLiX16g,415
+mm_std-0.3.26.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mm_std-0.3.26.dist-info/RECORD,,