githubkit 0.10.7__py3-none-any.whl → 0.11.0__py3-none-any.whl

githubkit/__init__.py

@@ -1,8 +1,13 @@
+from . import lazy_module
+
+lazy_module.apply()
+
 from .config import Config as Config
 from .github import GitHub as GitHub
 from .core import GitHubCore as GitHubCore
 from .response import Response as Response
 from .paginator import Paginator as Paginator
+from .compat import GitHubModel as GitHubModel
 from .auth import AppAuthStrategy as AppAuthStrategy
 from .auth import BaseAuthStrategy as BaseAuthStrategy
 from .auth import TokenAuthStrategy as TokenAuthStrategy

githubkit/auth/_url.py

@@ -26,21 +26,38 @@ APP_ROUTES = {
 }
 
 BYPASS_REGEX = re.compile(r"/login/(oauth/access_token|device/code)$")
-APP_AUTH_REGEX = re.compile(rf"(?:{'|'.join(APP_ROUTES)})[^/]*$", re.I)
+"""Regex to match routes that should bypass auth.
+
+See: https://github.com/octokit/auth-oauth-user.js/blob/b8b1141b21f285027e36a4a1670163e4db51d30f/src/hook.ts#L44
+"""
+APP_AUTH_REGEX = re.compile(rf"^(?:{'|'.join(APP_ROUTES)})$", re.I)
+"""Regex to match app authentication routes.
+
+See: https://github.com/octokit/auth-app.js/blob/c0068e06081a5d930799285a7c79c9c948b676b6/src/requires-app-auth.ts#L45
+"""
 BASIC_AUTH_REGEX = re.compile(r"/applications/[^/]+/(token|grant)s?")
+"""Regex to match basic authentication routes.
+
+See: https://github.com/octokit/auth-oauth-user.js/blob/b8b1141b21f285027e36a4a1670163e4db51d30f/src/requires-basic-auth.ts#L17
+"""
 OAUTH_BASE_REGEX = re.compile(r"^https://(api\.)?github\.com/?$")
 
 
 def require_bypass(url: httpx.URL) -> bool:
-    # bypass oauth web flow and device flow
+    """Check if the request should bypass auth."""
     return bool(BYPASS_REGEX.search(url.path))
 
 
 def require_app_auth(url: httpx.URL) -> bool:
+    """Check if the request should use app authentication.
+
+    Note: the input url must remove the base url first.
+    """
     return bool(APP_AUTH_REGEX.search(url.path))
 
 
 def require_basic_auth(url: httpx.URL) -> bool:
+    """Check if the request should use basic authentication."""
     return bool(BASIC_AUTH_REGEX.search(url.path))
 
 

githubkit/auth/app.py

@@ -7,12 +7,7 @@ import httpx
 from githubkit.exception import AuthCredentialError
 from githubkit.cache import DEFAULT_CACHE, BaseCache
 from githubkit.utils import UNSET, Unset, exclude_unset
-from githubkit.rest import (
-    BasicError,
-    ValidationError,
-    InstallationToken,
-    AppInstallationsInstallationIdAccessTokensPostBody,
-)
+from githubkit.compat import model_dump, type_validate_python
 
 from .base import BaseAuthStrategy
 from .oauth import OAuthAppAuthStrategy
@@ -25,7 +20,8 @@ except ImportError:
 
 if TYPE_CHECKING:
     from githubkit import Response, GitHubCore
-    from githubkit.rest.types import AppPermissionsType
+    from githubkit.versions.latest.models import InstallationToken
+    from githubkit.versions.latest.types import AppPermissionsType
 
 
 @dataclass
@@ -49,6 +45,16 @@ class AppAuth(httpx.Auth):
         "{installation_id}:{permissions}:{repositories}:{repository_ids}"
     )
 
+    def _get_api_route(self, url: httpx.URL) -> httpx.URL:
+        """Get the api route (path only) for the given url."""
+        base_url_path = self.github.config.base_url.path
+        if (path := url.path).startswith(base_url_path):
+            path = path[len(base_url_path) :]
+            # add leading slash
+            if not path.startswith("/"):
+                path = "/" + path
+        return httpx.URL(path=path)
+
     def _create_jwt(self) -> str:
         """Create a JWT authenticating as GitHub APP.
 
@@ -80,6 +86,10 @@ class AppAuth(httpx.Auth):
         return token
 
     def _build_installation_auth_request(self) -> httpx.Request:
+        from githubkit.versions.latest.models import (
+            AppInstallationsInstallationIdAccessTokensPostBody,
+        )
+
         if self.installation_id is UNSET:
             raise AuthCredentialError(
                 "GitHub APP installation_id must be provided "
@@ -91,13 +101,16 @@ class AppAuth(httpx.Auth):
             raw_path=base_url.raw_path
             + f"app/installations/{self.installation_id}/access_tokens".encode("ascii")
         )
-        body = AppInstallationsInstallationIdAccessTokensPostBody.parse_obj(
-            {
-                "repositories": self.repositories,
-                "repository_ids": self.repository_ids,
-                "permissions": self.permissions,
-            }
-        ).dict(by_alias=True)
+        body = model_dump(
+            type_validate_python(
+                AppInstallationsInstallationIdAccessTokensPostBody,
+                {
+                    "repositories": self.repositories,
+                    "repository_ids": self.repository_ids,
+                    "permissions": self.permissions,
+                },
+            )
+        )
         return httpx.Request(
             "POST",
             url,
@@ -111,6 +124,12 @@ class AppAuth(httpx.Auth):
     def _parse_installation_auth_response(
         self, response: httpx.Response
     ) -> "Response[InstallationToken]":
+        from githubkit.versions.latest.models import (
+            BasicError,
+            ValidationError,
+            InstallationToken,
+        )
+
         return self.github._check(
             response,
             response_model=InstallationToken,
@@ -150,7 +169,7 @@ class AppAuth(httpx.Auth):
         if require_bypass(request.url):
             yield request
             return
-        if require_app_auth(request.url):
+        if require_app_auth(self._get_api_route(request.url)):
             request.headers["Authorization"] = f"Bearer {self.get_jwt()}"
             yield request
             return
@@ -186,7 +205,7 @@ class AppAuth(httpx.Auth):
         if require_bypass(request.url):
             yield request
             return
-        if require_app_auth(request.url):
+        if require_app_auth(self._get_api_route(request.url)):
             request.headers["Authorization"] = f"Bearer {await self.aget_jwt()}"
             yield request
             return

githubkit/compat.py

@@ -0,0 +1,124 @@
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Type,
+    TypeVar,
+    Callable,
+    Protocol,
+    Generator,
+)
+
+from pydantic import VERSION
+
+T = TypeVar("T")
+
+PYDANTIC_V2 = int(VERSION.split(".", 1)[0]) == 2
+
+if TYPE_CHECKING:
+
+    class ModelBeforeValidator(Protocol):
+        def __call__(self, cls: Any, __value: Any) -> Any:
+            ...
+
+    class CustomValidationClass(Protocol):
+        @classmethod
+        def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+            ...
+
+    CVC = TypeVar("CVC", bound=CustomValidationClass)
+
+
+if PYDANTIC_V2:  # pragma: pydantic-v2
+    from pydantic_core import CoreSchema, core_schema
+    from pydantic_core import to_jsonable_python as to_jsonable_python
+    from pydantic import (
+        BaseModel,
+        ConfigDict,
+        TypeAdapter,
+        GetCoreSchemaHandler,
+        model_validator,
+    )
+
+    class GitHubModel(BaseModel):
+        model_config = ConfigDict(populate_by_name=True)
+
+    class ExtraGitHubModel(GitHubModel):
+        model_config = ConfigDict(extra="allow")
+
+    def type_validate_python(type_: Type[T], data: Any) -> T:
+        return TypeAdapter(type_).validate_python(data)
+
+    def type_validate_json(type_: Type[T], data: Any) -> T:
+        return TypeAdapter(type_).validate_json(data)
+
+    def model_dump(model: BaseModel, by_alias: bool = True) -> Dict[str, Any]:
+        return model.model_dump(by_alias=by_alias)
+
+    def model_rebuild(model: Type[BaseModel]):
+        model.model_rebuild()
+
+    def model_before_validator(func: "ModelBeforeValidator"):
+        return model_validator(mode="before")(func)
+
+    def __get_pydantic_core_schema__(
+        cls: Type["CustomValidationClass"],
+        source_type: Any,
+        handler: GetCoreSchemaHandler,
+    ) -> CoreSchema:
+        validators = list(cls.__get_validators__())
+        if len(validators) == 1:
+            return core_schema.no_info_plain_validator_function(validators[0])
+        return core_schema.chain_schema(
+            [core_schema.no_info_plain_validator_function(func) for func in validators]
+        )
+
+    def custom_validation(class_: Type["CVC"]) -> Type["CVC"]:
+        setattr(
+            class_,
+            "__get_pydantic_core_schema__",
+            classmethod(__get_pydantic_core_schema__),
+        )
+        return class_
+
+else:  # pragma: pydantic-v1
+    from pydantic.json import pydantic_encoder
+    from pydantic import Extra, BaseModel, parse_obj_as, parse_raw_as, root_validator
+
+    class GitHubModel(BaseModel):
+        class Config:
+            allow_population_by_field_name = True
+
+    class ExtraGitHubModel(BaseModel):
+        class Config:
+            extra = Extra.allow
+
+    def type_validate_python(type_: Type[T], data: Any) -> T:
+        return parse_obj_as(type_, data)
+
+    def type_validate_json(type_: Type[T], data: Any) -> T:
+        return parse_raw_as(type_, data)
+
+    def to_jsonable_python(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: to_jsonable_python(v) for k, v in obj.items()}
+
+        if isinstance(obj, (list, tuple)):
+            return [to_jsonable_python(item) for item in obj]
+
+        if obj is None or isinstance(obj, (int, float, str, bool)):
+            return obj
+
+        return pydantic_encoder(obj)
+
+    def model_dump(model: BaseModel, by_alias: bool = True) -> Dict[str, Any]:
+        return model.dict(by_alias=by_alias)
+
+    def model_rebuild(model: Type[BaseModel]):
+        return model.update_forward_refs()
+
+    def model_before_validator(func: "ModelBeforeValidator"):
+        return root_validator(pre=True)(func)
+
+    def custom_validation(class_: Type["CVC"]) -> Type["CVC"]:
+        return class_

githubkit/config.py

@@ -4,6 +4,9 @@ from typing import Any, Dict, List, Union, Optional
 
 import httpx
 
+from .retry import RETRY_DEFAULT
+from .typing import RetryDecisionFunc
+
 
 @dataclass(frozen=True)
 class Config:
@@ -12,6 +15,8 @@ class Config:
     user_agent: str
     follow_redirects: bool
     timeout: httpx.Timeout
+    http_cache: bool
+    auto_retry: Optional[RetryDecisionFunc]
 
     def dict(self) -> Dict[str, Any]:
         return asdict(self)
@@ -22,7 +27,11 @@ class Config:
 
 def build_base_url(base_url: Optional[Union[str, httpx.URL]]) -> httpx.URL:
     base_url = base_url or httpx.URL("https://api.github.com/")
-    return base_url if isinstance(base_url, httpx.URL) else httpx.URL(base_url)
+    base_url = base_url if isinstance(base_url, httpx.URL) else httpx.URL(base_url)
+    # enforce trailing slash
+    if not base_url.raw_path.endswith(b"/"):
+        base_url = base_url.copy_with(raw_path=base_url.raw_path + b"/")
+    return base_url
 
 
 def build_accept(
@@ -54,6 +63,17 @@ def build_timeout(
     return timeout if isinstance(timeout, httpx.Timeout) else httpx.Timeout(timeout)
 
 
+def build_auto_retry(
+    auto_retry: Union[bool, RetryDecisionFunc] = True
+) -> Optional[RetryDecisionFunc]:
+    if auto_retry is True:
+        return RETRY_DEFAULT
+    elif auto_retry:
+        return auto_retry
+    else:
+        return None
+
+
 def get_config(
     base_url: Optional[Union[str, httpx.URL]] = None,
     accept_format: Optional[str] = None,
@@ -61,6 +81,8 @@ def get_config(
     user_agent: Optional[str] = None,
     follow_redirects: bool = True,
     timeout: Optional[Union[float, httpx.Timeout]] = None,
+    http_cache: bool = True,
+    auto_retry: Union[bool, RetryDecisionFunc] = True,
 ) -> Config:
     return Config(
         build_base_url(base_url),
@@ -68,4 +90,6 @@ def get_config(
         build_user_agent(user_agent),
         follow_redirects,
         build_timeout(timeout),
+        http_cache,
+        build_auto_retry(auto_retry),
     )

githubkit/core.py

@@ -1,5 +1,7 @@
+import time
 from types import TracebackType
 from contextvars import ContextVar
+from datetime import datetime, timezone, timedelta
 from contextlib import contextmanager, asynccontextmanager
 from typing import (
     Any,
@@ -16,12 +18,13 @@ from typing import (
     overload,
 )
 
+import anyio
 import httpx
+import hishel
 
 from .response import Response
-from .utils import obj_to_jsonable
+from .compat import to_jsonable_python
 from .config import Config, get_config
-from .exception import RequestError, RequestFailed, RequestTimeout
 from .auth import BaseAuthStrategy, TokenAuthStrategy, UnauthAuthStrategy
 from .typing import (
     URLTypes,
@@ -30,6 +33,15 @@ from .typing import (
     ContentTypes,
     RequestFiles,
     QueryParamTypes,
+    RetryDecisionFunc,
+)
+from .exception import (
+    RequestError,
+    RequestFailed,
+    RequestTimeout,
+    GitHubException,
+    PrimaryRateLimitExceeded,
+    SecondaryRateLimitExceeded,
 )
 
 T = TypeVar("T")
@@ -79,6 +91,8 @@ class GitHubCore(Generic[A]):
         user_agent: Optional[str] = None,
         follow_redirects: bool = True,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        http_cache: bool = True,
+        auto_retry: Union[bool, RetryDecisionFunc] = True,
     ):
         ...
 
@@ -94,6 +108,8 @@ class GitHubCore(Generic[A]):
         user_agent: Optional[str] = None,
         follow_redirects: bool = True,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        http_cache: bool = True,
+        auto_retry: Union[bool, RetryDecisionFunc] = True,
     ):
         ...
 
@@ -109,6 +125,8 @@ class GitHubCore(Generic[A]):
         user_agent: Optional[str] = None,
         follow_redirects: bool = True,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        http_cache: bool = True,
+        auto_retry: Union[bool, RetryDecisionFunc] = True,
     ):
         ...
 
@@ -123,6 +141,8 @@ class GitHubCore(Generic[A]):
         user_agent: Optional[str] = None,
         follow_redirects: bool = True,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        http_cache: bool = True,
+        auto_retry: Union[bool, RetryDecisionFunc] = True,
     ):
         auth = auth or UnauthAuthStrategy()  # type: ignore
         self.auth: A = (  # type: ignore
@@ -130,7 +150,14 @@ class GitHubCore(Generic[A]):
         )
 
         self.config = config or get_config(
-            base_url, accept_format, previews, user_agent, follow_redirects, timeout
+            base_url,
+            accept_format,
+            previews,
+            user_agent,
+            follow_redirects,
+            timeout,
+            http_cache,
+            auto_retry,
         )
 
         self.__sync_client: ContextVar[Optional[httpx.Client]] = ContextVar(
@@ -187,7 +214,14 @@ class GitHubCore(Generic[A]):
 
     # create sync client
     def _create_sync_client(self) -> httpx.Client:
-        return httpx.Client(**self._get_client_defaults())
+        if self.config.http_cache:
+            transport = hishel.CacheTransport(
+                httpx.HTTPTransport(), storage=hishel.InMemoryStorage()
+            )
+        else:
+            transport = httpx.HTTPTransport()
+
+        return httpx.Client(**self._get_client_defaults(), transport=transport)
 
     # get or create sync client
     @contextmanager
@@ -203,7 +237,14 @@ class GitHubCore(Generic[A]):
 
     # create async client
     def _create_async_client(self) -> httpx.AsyncClient:
-        return httpx.AsyncClient(**self._get_client_defaults())
+        if self.config.http_cache:
+            transport = hishel.AsyncCacheTransport(
+                httpx.AsyncHTTPTransport(), storage=hishel.AsyncInMemoryStorage()
+            )
+        else:
+            transport = httpx.AsyncHTTPTransport()
+
+        return httpx.AsyncClient(**self._get_client_defaults(), transport=transport)
 
     # get or create async client
     @asynccontextmanager
@@ -240,7 +281,7 @@ class GitHubCore(Generic[A]):
                     content=content,
                     data=data,
                     files=files,
-                    json=obj_to_jsonable(json),
+                    json=to_jsonable_python(json),
                     headers=headers,
                     cookies=cookies,
                 )
@@ -272,7 +313,7 @@ class GitHubCore(Generic[A]):
                     content=content,
                     data=data,
                     files=files,
-                    json=obj_to_jsonable(json),
+                    json=to_jsonable_python(json),
                     headers=headers,
                     cookies=cookies,
                 )
@@ -291,15 +332,73 @@ class GitHubCore(Generic[A]):
         if response.is_error:
             error_models = error_models or {}
             status_code = str(response.status_code)
+
             error_model = error_models.get(
                 status_code,
                 error_models.get(
                     f"{status_code[:-2]}XX", error_models.get("default", Any)
                 ),
             )
-            rep = Response(response, error_model)
-            raise RequestFailed(rep)
-        return Response(response, response_model)
+            resp = Response(response, error_model)
+        else:
+            resp = Response(response, response_model)
+
+        # only check rate limit when response is 403 or 429
+        if response.status_code in (403, 429):
+            self._check_rate_limit(resp)
+
+        if response.is_error:
+            raise RequestFailed(resp)
+        return resp
+
+    # check rate limit
+    def _check_rate_limit(self, response: Response) -> None:
+        # check rate limit exceeded
+        # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#exceeding-the-rate-limit
+        # https://docs.github.com/en/graphql/overview/rate-limits-and-node-limits-for-the-graphql-api#exceeding-the-rate-limit
+        # https://github.com/octokit/plugin-throttling.js/blob/135a0f556752a6c4c0ed3b2798bb58e228cd179a/src/index.ts#L134-L179
+
+        # Secondary rate limits
+        # the `retry-after` response header is present
+        if "retry-after" in response.headers:
+            raise SecondaryRateLimitExceeded(
+                response, self._extract_retry_after(response)
+            )
+
+        if (
+            "x-ratelimit-remaining" in response.headers
+            and response.headers["x-ratelimit-remaining"] == "0"
+        ):
+            retry_after = self._extract_retry_after(response)
+
+            try:
+                error = response.json()
+            except Exception:
+                error = None
+
+            # Secondary rate limits
+            # error message indicates that you exceeded a secondary rate limit
+            if (
+                isinstance(error, dict)
+                and "message" in error
+                and "secondary rate" in error["message"]
+            ):
+                raise SecondaryRateLimitExceeded(response, retry_after)
+
+            # Primary rate limits
+            raise PrimaryRateLimitExceeded(response, retry_after)
+
+    def _extract_retry_after(self, response: Response) -> timedelta:
+        if "retry-after" in response.headers:
+            return timedelta(seconds=int(response.headers["retry-after"]))
+        elif "x-ratelimit-reset" in response.headers:
+            retry_after = datetime.fromtimestamp(
+                int(response.headers["x-ratelimit-reset"]), tz=timezone.utc
+            ) - datetime.now(tz=timezone.utc)
+            return max(retry_after, timedelta())
+        else:
+            # wait for at least one minute before retrying
+            return timedelta(seconds=60)
 
     # sync request and check
     def request(
@@ -317,18 +416,32 @@ class GitHubCore(Generic[A]):
         response_model: Type[T] = Any,
         error_models: Optional[Dict[str, type]] = None,
     ) -> Response[T]:
-        raw_resp = self._request(
-            method,
-            url,
-            params=params,
-            content=content,
-            data=data,
-            files=files,
-            json=json,
-            headers=headers,
-            cookies=cookies,
-        )
-        return self._check(raw_resp, response_model, error_models)
+        retry_count: int = 0
+        while True:
+            try:
+                raw_resp = self._request(
+                    method,
+                    url,
+                    params=params,
+                    content=content,
+                    data=data,
+                    files=files,
+                    json=json,
+                    headers=headers,
+                    cookies=cookies,
+                )
+                return self._check(raw_resp, response_model, error_models)
+            except GitHubException as e:
+                if self.config.auto_retry is None:
+                    raise
+                else:
+                    do_retry, retry_after = self.config.auto_retry(e, retry_count)
+
+                if not do_retry:
+                    raise
+
+                time.sleep(retry_after.total_seconds() if retry_after else 60)
+                retry_count += 1
 
     # async request and check
     async def arequest(
@@ -346,15 +459,29 @@ class GitHubCore(Generic[A]):
         response_model: Type[T] = Any,
         error_models: Optional[Dict[str, type]] = None,
     ) -> Response[T]:
-        raw_resp = await self._arequest(
-            method,
-            url,
-            params=params,
-            content=content,
-            data=data,
-            files=files,
-            json=json,
-            headers=headers,
-            cookies=cookies,
-        )
-        return self._check(raw_resp, response_model, error_models)
+        retry_count: int = 0
+        while True:
+            try:
+                raw_resp = await self._arequest(
+                    method,
+                    url,
+                    params=params,
+                    content=content,
+                    data=data,
+                    files=files,
+                    json=json,
+                    headers=headers,
+                    cookies=cookies,
+                )
+                return self._check(raw_resp, response_model, error_models)
+            except GitHubException as e:
+                if self.config.auto_retry is None:
+                    raise
+                else:
+                    do_retry, retry_after = self.config.auto_retry(e, retry_count)
+
+                if not do_retry:
+                    raise
+
+                await anyio.sleep(retry_after.total_seconds() if retry_after else 60)
+                retry_count += 1

githubkit/exception.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from typing import TYPE_CHECKING
 
 import httpx
@@ -50,6 +51,29 @@ class RequestFailed(GitHubException):
         )
 
 
+class RateLimitExceeded(RequestFailed):
+    """API request failed with rate limit exceeded"""
+
+    def __init__(self, response: "Response", retry_after: timedelta):
+        super().__init__(response)
+        self.retry_after = retry_after
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(method={self.request.method}, "
+            f"url={self.request.url}, status_code={self.response.status_code}, "
+            f"retry_after={self.retry_after})"
+        )
+
+
+class PrimaryRateLimitExceeded(RateLimitExceeded):
+    """API request failed with primary rate limit exceeded"""
+
+
+class SecondaryRateLimitExceeded(RateLimitExceeded):
+    """API request failed with secondary rate limit exceeded"""
+
+
 class GraphQLFailed(GitHubException):
     """GraphQL request with errors in response"""