huggingface-hub 0.36.0rc0__py3-none-any.whl → 1.0.0__py3-none-any.whl

huggingface_hub/utils/_http.py

@@ -12,22 +12,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Contains utilities to handle HTTP requests in Huggingface Hub."""
+"""Contains utilities to handle HTTP requests in huggingface_hub."""
 
+import atexit
 import io
-import os
+import json
 import re
 import threading
 import time
 import uuid
-from functools import lru_cache
+from contextlib import contextmanager
+from http import HTTPStatus
 from shlex import quote
-from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Generator, Optional, Union
 
-import requests
-from requests import HTTPError, Response
-from requests.adapters import HTTPAdapter
-from requests.models import PreparedRequest
+import httpx
 
 from huggingface_hub.errors import OfflineModeIsEnabled
 
@@ -35,14 +34,13 @@ from .. import constants
 from ..errors import (
     BadRequestError,
     DisabledRepoError,
-    EntryNotFoundError,
     GatedRepoError,
     HfHubHTTPError,
+    RemoteEntryNotFoundError,
     RepositoryNotFoundError,
     RevisionNotFoundError,
 )
 from . import logging
-from ._fixes import JSONDecodeError
 from ._lfs import SliceFileObj
 from ._typing import HTTP_METHOD_T
 
@@ -71,142 +69,266 @@ REPO_API_REGEX = re.compile(
 )
 
 
-class UniqueRequestIdAdapter(HTTPAdapter):
-    X_AMZN_TRACE_ID = "X-Amzn-Trace-Id"
+def hf_request_event_hook(request: httpx.Request) -> None:
+    """
+    Event hook that will be used to make HTTP requests to the Hugging Face Hub.
 
-    def add_headers(self, request, **kwargs):
-        super().add_headers(request, **kwargs)
+    What it does:
+    - Block requests if offline mode is enabled
+    - Add a request ID to the request headers
+    - Log the request if debug mode is enabled
+    """
+    if constants.HF_HUB_OFFLINE:
+        raise OfflineModeIsEnabled(
+            f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable."
+        )
 
-        # Add random request ID => easier for server-side debug
-        if X_AMZN_TRACE_ID not in request.headers:
-            request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4())
+    # Add random request ID => easier for server-side debugging
+    if X_AMZN_TRACE_ID not in request.headers:
+        request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4())
+    request_id = request.headers.get(X_AMZN_TRACE_ID)
 
-        # Add debug log
-        has_token = len(str(request.headers.get("authorization", ""))) > 0
-        logger.debug(
-            f"Request {request.headers[X_AMZN_TRACE_ID]}: {request.method} {request.url} (authenticated: {has_token})"
-        )
+    # Debug log
+    logger.debug(
+        "Request %s: %s %s (authenticated: %s)",
+        request_id,
+        request.method,
+        request.url,
+        request.headers.get("authorization") is not None,
+    )
+    if constants.HF_DEBUG:
+        logger.debug("Send: %s", _curlify(request))
 
-    def send(self, request: PreparedRequest, *args, **kwargs) -> Response:
-        """Catch any RequestException to append request id to the error message for debugging."""
-        if constants.HF_DEBUG:
-            logger.debug(f"Send: {_curlify(request)}")
-        try:
-            return super().send(request, *args, **kwargs)
-        except requests.RequestException as e:
-            request_id = request.headers.get(X_AMZN_TRACE_ID)
-            if request_id is not None:
-                # Taken from https://stackoverflow.com/a/58270258
-                e.args = (*e.args, f"(Request ID: {request_id})")
-            raise
+    return request_id
 
 
-class OfflineAdapter(HTTPAdapter):
-    def send(self, request: PreparedRequest, *args, **kwargs) -> Response:
-        raise OfflineModeIsEnabled(
-            f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable."
-        )
+async def async_hf_request_event_hook(request: httpx.Request) -> None:
+    """
+    Async version of `hf_request_event_hook`.
+    """
+    return hf_request_event_hook(request)
 
 
-def _default_backend_factory() -> requests.Session:
-    session = requests.Session()
-    if constants.HF_HUB_OFFLINE:
-        session.mount("http://", OfflineAdapter())
-        session.mount("https://", OfflineAdapter())
-    else:
-        session.mount("http://", UniqueRequestIdAdapter())
-        session.mount("https://", UniqueRequestIdAdapter())
-    return session
+async def async_hf_response_event_hook(response: httpx.Response) -> None:
+    if response.status_code >= 400:
+        # If response will raise, read content from stream to have it available when raising the exception
+        # If content-length is not set or is too large, skip reading the content to avoid OOM
+        if "Content-length" in response.headers:
+            try:
+                length = int(response.headers["Content-length"])
+            except ValueError:
+                return
+
+            if length < 1_000_000:
+                await response.aread()
 
 
-BACKEND_FACTORY_T = Callable[[], requests.Session]
-_GLOBAL_BACKEND_FACTORY: BACKEND_FACTORY_T = _default_backend_factory
+def default_client_factory() -> httpx.Client:
+    """
+    Factory function to create a `httpx.Client` with the default transport.
+    """
+    return httpx.Client(
+        event_hooks={"request": [hf_request_event_hook]},
+        follow_redirects=True,
+        timeout=httpx.Timeout(constants.DEFAULT_REQUEST_TIMEOUT, write=60.0),
+    )
 
 
-def configure_http_backend(backend_factory: BACKEND_FACTORY_T = _default_backend_factory) -> None:
+def default_async_client_factory() -> httpx.AsyncClient:
+    """
+    Factory function to create a `httpx.AsyncClient` with the default transport.
     """
-    Configure the HTTP backend by providing a `backend_factory`. Any HTTP calls made by `huggingface_hub` will use a
-    Session object instantiated by this factory. This can be useful if you are running your scripts in a specific
-    environment requiring custom configuration (e.g. custom proxy or certifications).
+    return httpx.AsyncClient(
+        event_hooks={"request": [async_hf_request_event_hook], "response": [async_hf_response_event_hook]},
+        follow_redirects=True,
+        timeout=httpx.Timeout(constants.DEFAULT_REQUEST_TIMEOUT, write=60.0),
+    )
 
-    Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe,
-    `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory`
-    set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between
-    calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned.
 
-    See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`.
+CLIENT_FACTORY_T = Callable[[], httpx.Client]
+ASYNC_CLIENT_FACTORY_T = Callable[[], httpx.AsyncClient]
 
-    Example:
-    ```py
-    import requests
-    from huggingface_hub import configure_http_backend, get_session
+_CLIENT_LOCK = threading.Lock()
+_GLOBAL_CLIENT_FACTORY: CLIENT_FACTORY_T = default_client_factory
+_GLOBAL_ASYNC_CLIENT_FACTORY: ASYNC_CLIENT_FACTORY_T = default_async_client_factory
+_GLOBAL_CLIENT: Optional[httpx.Client] = None
 
-    # Create a factory function that returns a Session with configured proxies
-    def backend_factory() -> requests.Session:
-        session = requests.Session()
-        session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"}
-        return session
 
-    # Set it as the default session factory
-    configure_http_backend(backend_factory=backend_factory)
+def set_client_factory(client_factory: CLIENT_FACTORY_T) -> None:
+    """
+    Set the HTTP client factory to be used by `huggingface_hub`.
 
-    # In practice, this is mostly done internally in `huggingface_hub`
-    session = get_session()
-    ```
+    The client factory is a method that returns a `httpx.Client` object. On the first call to [`get_client`] the client factory
+    will be used to create a new `httpx.Client` object that will be shared between all calls made by `huggingface_hub`.
+
+    This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
+
+    Use [`get_client`] to get a correctly configured `httpx.Client`.
     """
-    global _GLOBAL_BACKEND_FACTORY
-    _GLOBAL_BACKEND_FACTORY = backend_factory
-    reset_sessions()
+    global _GLOBAL_CLIENT_FACTORY
+    with _CLIENT_LOCK:
+        close_session()
+        _GLOBAL_CLIENT_FACTORY = client_factory
 
 
-def get_session() -> requests.Session:
+def set_async_client_factory(async_client_factory: ASYNC_CLIENT_FACTORY_T) -> None:
     """
-    Get a `requests.Session` object, using the session factory from the user.
+    Set the HTTP async client factory to be used by `huggingface_hub`.
 
-    Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe,
-    `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory`
-    set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between
-    calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned.
+    The async client factory is a method that returns a `httpx.AsyncClient` object.
+    This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
+    Use [`get_async_client`] to get a correctly configured `httpx.AsyncClient`.
 
-    See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`.
+    <Tip warning={true}>
 
-    Example:
-    ```py
-    import requests
-    from huggingface_hub import configure_http_backend, get_session
+    Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
+    It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
+
+    </Tip>
+    """
+    global _GLOBAL_ASYNC_CLIENT_FACTORY
+    _GLOBAL_ASYNC_CLIENT_FACTORY = async_client_factory
 
-    # Create a factory function that returns a Session with configured proxies
-    def backend_factory() -> requests.Session:
-        session = requests.Session()
-        session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"}
-        return session
 
-    # Set it as the default session factory
-    configure_http_backend(backend_factory=backend_factory)
+def get_session() -> httpx.Client:
+    """
+    Get a `httpx.Client` object, using the transport factory from the user.
 
-    # In practice, this is mostly done internally in `huggingface_hub`
-    session = get_session()
-    ```
+    This client is shared between all calls made by `huggingface_hub`. Therefore you should not close it manually.
+
+    Use [`set_client_factory`] to customize the `httpx.Client`.
+    """
+    global _GLOBAL_CLIENT
+    if _GLOBAL_CLIENT is None:
+        with _CLIENT_LOCK:
+            _GLOBAL_CLIENT = _GLOBAL_CLIENT_FACTORY()
+    return _GLOBAL_CLIENT
+
+
+def get_async_session() -> httpx.AsyncClient:
     """
-    return _get_session_from_cache(process_id=os.getpid(), thread_id=threading.get_ident())
+    Return a `httpx.AsyncClient` object, using the transport factory from the user.
+
+    Use [`set_async_client_factory`] to customize the `httpx.AsyncClient`.
 
+    <Tip warning={true}>
 
-def reset_sessions() -> None:
-    """Reset the cache of sessions.
+    Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
+    It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
 
-    Mostly used internally when sessions are reconfigured or an SSLError is raised.
-    See [`configure_http_backend`] for more details.
+    </Tip>
     """
-    _get_session_from_cache.cache_clear()
+    return _GLOBAL_ASYNC_CLIENT_FACTORY()
 
 
-@lru_cache
-def _get_session_from_cache(process_id: int, thread_id: int) -> requests.Session:
+def close_session() -> None:
     """
-    Create a new session per thread using global factory. Using LRU cache (maxsize 128) to avoid memory leaks when
-    using thousands of threads. Cache is cleared when `configure_http_backend` is called.
+    Close the global `httpx.Client` used by `huggingface_hub`.
+
+    If a Client is closed, it will be recreated on the next call to [`get_session`].
+
+    Can be useful if e.g. an SSL certificate has been updated.
     """
-    return _GLOBAL_BACKEND_FACTORY()
+    global _GLOBAL_CLIENT
+    client = _GLOBAL_CLIENT
+
+    # First, set global client to None
+    _GLOBAL_CLIENT = None
+
+    # Then, close the clients
+    if client is not None:
+        try:
+            client.close()
+        except Exception as e:
+            logger.warning(f"Error closing client: {e}")
+
+
+atexit.register(close_session)
+
+
+def _http_backoff_base(
+    method: HTTP_METHOD_T,
+    url: str,
+    *,
+    max_retries: int = 5,
+    base_wait_time: float = 1,
+    max_wait_time: float = 8,
+    retry_on_exceptions: Union[type[Exception], tuple[type[Exception], ...]] = (
+        httpx.TimeoutException,
+        httpx.NetworkError,
+    ),
+    retry_on_status_codes: Union[int, tuple[int, ...]] = HTTPStatus.SERVICE_UNAVAILABLE,
+    stream: bool = False,
+    **kwargs,
+) -> Generator[httpx.Response, None, None]:
+    """Internal implementation of HTTP backoff logic shared between `http_backoff` and `http_stream_backoff`."""
+    if isinstance(retry_on_exceptions, type):  # Tuple from single exception type
+        retry_on_exceptions = (retry_on_exceptions,)
+
+    if isinstance(retry_on_status_codes, int):  # Tuple from single status code
+        retry_on_status_codes = (retry_on_status_codes,)
+
+    nb_tries = 0
+    sleep_time = base_wait_time
+
+    # If `data` is used and is a file object (or any IO), it will be consumed on the
+    # first HTTP request. We need to save the initial position so that the full content
+    # of the file is re-sent on http backoff. See warning tip in docstring.
+    io_obj_initial_pos = None
+    if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)):
+        io_obj_initial_pos = kwargs["data"].tell()
+
+    client = get_session()
+    while True:
+        nb_tries += 1
+        try:
+            # If `data` is used and is a file object (or any IO), set back cursor to
+            # initial position.
+            if io_obj_initial_pos is not None:
+                kwargs["data"].seek(io_obj_initial_pos)
+
+            # Perform request and handle response
+            def _should_retry(response: httpx.Response) -> bool:
+                """Handle response and return True if should retry, False if should return/yield."""
+                if response.status_code not in retry_on_status_codes:
+                    return False  # Success, don't retry
+
+                # Wrong status code returned (HTTP 503 for instance)
+                logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}")
+                if nb_tries > max_retries:
+                    hf_raise_for_status(response)  # Will raise uncaught exception
+                    # Return/yield response to avoid infinite loop in the corner case where the
+                    # user ask for retry on a status code that doesn't raise_for_status.
+                    return False  # Don't retry, return/yield response
+
+                return True  # Should retry
+
+            if stream:
+                with client.stream(method=method, url=url, **kwargs) as response:
+                    if not _should_retry(response):
+                        yield response
+                        return
+            else:
+                response = client.request(method=method, url=url, **kwargs)
+                if not _should_retry(response):
+                    yield response
+                    return
+
+        except retry_on_exceptions as err:
+            logger.warning(f"'{err}' thrown while requesting {method} {url}")
+
+            if isinstance(err, httpx.ConnectError):
+                close_session()  # In case of SSLError it's best to close the shared httpx.Client objects
+
+            if nb_tries > max_retries:
+                raise err
+
+        # Sleep for X seconds
+        logger.warning(f"Retrying in {sleep_time}s [Retry {nb_tries}/{max_retries}].")
+        time.sleep(sleep_time)
+
+        # Update sleep time for next retry
+        sleep_time = min(max_wait_time, sleep_time * 2)  # Exponential backoff
 
 
 def http_backoff(
@@ -216,15 +338,14 @@ def http_backoff(
     max_retries: int = 5,
     base_wait_time: float = 1,
     max_wait_time: float = 8,
-    retry_on_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = (
-        requests.Timeout,
-        requests.ConnectionError,
-        requests.exceptions.ChunkedEncodingError,
+    retry_on_exceptions: Union[type[Exception], tuple[type[Exception], ...]] = (
+        httpx.TimeoutException,
+        httpx.NetworkError,
     ),
-    retry_on_status_codes: Union[int, Tuple[int, ...]] = (500, 502, 503, 504),
+    retry_on_status_codes: Union[int, tuple[int, ...]] = HTTPStatus.SERVICE_UNAVAILABLE,
     **kwargs,
-) -> Response:
-    """Wrapper around requests to retry calls on an endpoint, with exponential backoff.
+) -> httpx.Response:
+    """Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
 
     Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
     and/or on specific status codes (ex: service unavailable). If the call failed more
@@ -247,19 +368,20 @@ def http_backoff(
             `max_wait_time`.
         max_wait_time (`float`, *optional*, defaults to `8`):
             Maximum duration (in seconds) to wait before retrying.
-        retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*):
+        retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
             Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
-            By default, retry on `requests.Timeout`, `requests.ConnectionError` and `requests.exceptions.ChunkedEncodingError`.
-        retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `(500, 502, 503, 504)`):
-            Define on which status codes the request must be retried. By default, 5xx errors are retried.
+            By default, retry on `httpx.TimeoutException` and `httpx.NetworkError`.
+        retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `503`):
+            Define on which status codes the request must be retried. By default, only
+            HTTP 503 Service Unavailable is retried.
         **kwargs (`dict`, *optional*):
-            kwargs to pass to `requests.request`.
+            kwargs to pass to `httpx.request`.
 
     Example:
     ```
     >>> from huggingface_hub.utils import http_backoff
 
-    # Same usage as "requests.request".
+    # Same usage as "httpx.request".
     >>> response = http_backoff("GET", "https://www.google.com")
     >>> response.raise_for_status()
 
@@ -277,59 +399,105 @@ def http_backoff(
     > will fail. If this is a hard constraint for you, please let us know by opening an
     > issue on [Github](https://github.com/huggingface/huggingface_hub).
     """
-    if isinstance(retry_on_exceptions, type):  # Tuple from single exception type
-        retry_on_exceptions = (retry_on_exceptions,)
+    return next(
+        _http_backoff_base(
+            method=method,
+            url=url,
+            max_retries=max_retries,
+            base_wait_time=base_wait_time,
+            max_wait_time=max_wait_time,
+            retry_on_exceptions=retry_on_exceptions,
+            retry_on_status_codes=retry_on_status_codes,
+            stream=False,
+            **kwargs,
+        )
+    )
 
-    if isinstance(retry_on_status_codes, int):  # Tuple from single status code
-        retry_on_status_codes = (retry_on_status_codes,)
 
-    nb_tries = 0
-    sleep_time = base_wait_time
+@contextmanager
+def http_stream_backoff(
+    method: HTTP_METHOD_T,
+    url: str,
+    *,
+    max_retries: int = 5,
+    base_wait_time: float = 1,
+    max_wait_time: float = 8,
+    retry_on_exceptions: Union[type[Exception], tuple[type[Exception], ...]] = (
+        httpx.TimeoutException,
+        httpx.NetworkError,
+    ),
+    retry_on_status_codes: Union[int, tuple[int, ...]] = HTTPStatus.SERVICE_UNAVAILABLE,
+    **kwargs,
+) -> Generator[httpx.Response, None, None]:
+    """Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
 
-    # If `data` is used and is a file object (or any IO), it will be consumed on the
-    # first HTTP request. We need to save the initial position so that the full content
-    # of the file is re-sent on http backoff. See warning tip in docstring.
-    io_obj_initial_pos = None
-    if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)):
-        io_obj_initial_pos = kwargs["data"].tell()
+    Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
+    and/or on specific status codes (ex: service unavailable). If the call failed more
+    than `max_retries`, the exception is thrown or `raise_for_status` is called on the
+    response object.
 
-    session = get_session()
-    while True:
-        nb_tries += 1
-        try:
-            # If `data` is used and is a file object (or any IO), set back cursor to
-            # initial position.
-            if io_obj_initial_pos is not None:
-                kwargs["data"].seek(io_obj_initial_pos)
+    Re-implement mechanisms from the `backoff` library to avoid adding an external
+    dependencies to `hugging_face_hub`. See https://github.com/litl/backoff.
 
-            # Perform request and return if status_code is not in the retry list.
-            response = session.request(method=method, url=url, **kwargs)
-            if response.status_code not in retry_on_status_codes:
-                return response
+    Args:
+        method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`):
+            HTTP method to perform.
+        url (`str`):
+            The URL of the resource to fetch.
+        max_retries (`int`, *optional*, defaults to `5`):
+            Maximum number of retries, defaults to 5 (no retries).
+        base_wait_time (`float`, *optional*, defaults to `1`):
+            Duration (in seconds) to wait before retrying the first time.
+            Wait time between retries then grows exponentially, capped by
+            `max_wait_time`.
+        max_wait_time (`float`, *optional*, defaults to `8`):
+            Maximum duration (in seconds) to wait before retrying.
+        retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
+            Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
+            By default, retry on `httpx.Timeout` and `httpx.NetworkError`.
+        retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `503`):
+            Define on which status codes the request must be retried. By default, only
+            HTTP 503 Service Unavailable is retried.
+        **kwargs (`dict`, *optional*):
+            kwargs to pass to `httpx.request`.
 
-            # Wrong status code returned (HTTP 503 for instance)
-            logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}")
-            if nb_tries > max_retries:
-                response.raise_for_status()  # Will raise uncaught exception
-                # We return response to avoid infinite loop in the corner case where the
-                # user ask for retry on a status code that doesn't raise_for_status.
-                return response
+    Example:
+    ```
+    >>> from huggingface_hub.utils import http_stream_backoff
 
-        except retry_on_exceptions as err:
-            logger.warning(f"'{err}' thrown while requesting {method} {url}")
+    # Same usage as "httpx.stream".
+    >>> with http_stream_backoff("GET", "https://www.google.com") as response:
+    ...     for chunk in response.iter_bytes():
+    ...         print(chunk)
 
-            if isinstance(err, requests.ConnectionError):
-                reset_sessions()  # In case of SSLError it's best to reset the shared requests.Session objects
+    # If you expect a Gateway Timeout from time to time
+    >>> with http_stream_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) as response:
+    ...     response.raise_for_status()
+    ```
 
-            if nb_tries > max_retries:
-                raise err
+    <Tip warning={true}>
 
-        # Sleep for X seconds
-        logger.warning(f"Retrying in {sleep_time}s [Retry {nb_tries}/{max_retries}].")
-        time.sleep(sleep_time)
+    When using `httpx` it is possible to stream data by passing an iterator to the
+    `data` argument. On http backoff this is a problem as the iterator is not reset
+    after a failed call. This issue is mitigated for file objects or any IO streams
+    by saving the initial position of the cursor (with `data.tell()`) and resetting the
+    cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff
+    will fail. If this is a hard constraint for you, please let us know by opening an
+    issue on [Github](https://github.com/huggingface/huggingface_hub).
 
-        # Update sleep time for next retry
-        sleep_time = min(max_wait_time, sleep_time * 2)  # Exponential backoff
+    </Tip>
+    """
+    yield from _http_backoff_base(
+        method=method,
+        url=url,
+        max_retries=max_retries,
+        base_wait_time=base_wait_time,
+        max_wait_time=max_wait_time,
+        retry_on_exceptions=retry_on_exceptions,
+        retry_on_status_codes=retry_on_status_codes,
+        stream=True,
+        **kwargs,
+    )
 
 
 def fix_hf_endpoint_in_url(url: str, endpoint: Optional[str]) -> str:
@@ -345,38 +513,18 @@ def fix_hf_endpoint_in_url(url: str, endpoint: Optional[str]) -> str:
     return url
 
 
-def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None) -> None:
+def hf_raise_for_status(response: httpx.Response, endpoint_name: Optional[str] = None) -> None:
     """
-    Internal version of `response.raise_for_status()` that will refine a
-    potential HTTPError. Raised exception will be an instance of `HfHubHTTPError`.
-
-    This helper is meant to be the unique method to raise_for_status when making a call
-    to the Hugging Face Hub.
-
-
-    Example:
-    ```py
-        import requests
-        from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError
+    Internal version of `response.raise_for_status()` that will refine a potential HTTPError.
+    Raised exception will be an instance of [`~errors.HfHubHTTPError`].
 
-        response = get_session().post(...)
-        try:
-            hf_raise_for_status(response)
-        except HfHubHTTPError as e:
-            print(str(e)) # formatted message
-            e.request_id, e.server_message # details returned by server
-
-            # Complete the error message with additional information once it's raised
-            e.append_to_message("\n`create_commit` expects the repository to exist.")
-            raise
-    ```
+    This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub.
 
     Args:
         response (`Response`):
             Response from the server.
         endpoint_name (`str`, *optional*):
-            Name of the endpoint that has been called. If provided, the error message
-            will be more complete.
+            Name of the endpoint that has been called. If provided, the error message will be more complete.
 
     > [!WARNING]
     > Raises when the request has failed:
@@ -400,7 +548,10 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
     """
     try:
         response.raise_for_status()
-    except HTTPError as e:
+    except httpx.HTTPStatusError as e:
+        if response.status_code // 100 == 3:
+            return  # Do not raise on redirects to stay consistent with `requests`
+
         error_code = response.headers.get("X-Error-Code")
         error_message = response.headers.get("X-Error-Message")
 
@@ -410,7 +561,7 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
 
         elif error_code == "EntryNotFound":
             message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}."
-            raise _format(EntryNotFoundError, message, response) from e
+            raise _format(RemoteEntryNotFoundError, message, response) from e
 
         elif error_code == "GatedRepo":
             message = (
@@ -433,7 +584,7 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
             and error_message != "Invalid credentials in Authorization header"
             and response.request is not None
             and response.request.url is not None
-            and REPO_API_REGEX.search(response.request.url) is not None
+            and REPO_API_REGEX.search(str(response.request.url)) is not None
         ):
             # 401 is misleading as it is returned for:
             #    - private and gated repos if user is not authenticated
@@ -475,7 +626,7 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
         raise _format(HfHubHTTPError, str(e), response) from e
 
 
-def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Response) -> HfHubHTTPError:
+def _format(error_type: type[HfHubHTTPError], custom_message: str, response: httpx.Response) -> HfHubHTTPError:
     server_errors = []
 
     # Retrieve server error from header
@@ -486,7 +637,19 @@ def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Res
     # Retrieve server error from body
     try:
         # Case errors are returned in a JSON format
-        data = response.json()
+        try:
+            data = response.json()
+        except httpx.ResponseNotRead:
+            try:
+                response.read()  # In case of streaming response, we need to read the response first
+                data = response.json()
+            except RuntimeError:
+                # In case of async streaming response, we can't read the stream here.
+                # In practice if user is using the default async client from `get_async_client`, the stream will have
+                # already been read in the async event hook `async_hf_response_event_hook`.
+                #
+                # Here, we are skipping reading the response to avoid RuntimeError but it happens only if async + stream + used httpx.AsyncClient directly.
+                data = {}
 
         error = data.get("error")
         if error is not None:
@@ -504,7 +667,7 @@ def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Res
                 if "message" in error:
                     server_errors.append(error["message"])
 
-    except JSONDecodeError:
+    except json.JSONDecodeError:
         # If content is not JSON and not HTML, append the text
         content_type = response.headers.get("Content-Type", "")
         if response.text and "html" not in content_type.lower():
@@ -549,15 +712,15 @@ def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Res
     return error_type(final_error_message.strip(), response=response, server_message=server_message or None)
 
 
-def _curlify(request: requests.PreparedRequest) -> str:
-    """Convert a `requests.PreparedRequest` into a curl command (str).
+def _curlify(request: httpx.Request) -> str:
+    """Convert a `httpx.Request` into a curl command (str).
 
     Used for debug purposes only.
 
     Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py.
     MIT License Copyright (c) 2016 Egor.
     """
-    parts: List[Tuple[Any, Any]] = [
+    parts: list[tuple[Any, Any]] = [
         ("curl", None),
         ("-X", request.method),
     ]
@@ -565,16 +728,16 @@ def _curlify(request: requests.PreparedRequest) -> str:
     for k, v in sorted(request.headers.items()):
         if k.lower() == "authorization":
             v = "<TOKEN>"  # Hide authorization header, no matter its value (can be Bearer, Key, etc.)
-        parts += [("-H", "{0}: {1}".format(k, v))]
-
-    if request.body:
-        body = request.body
-        if isinstance(body, bytes):
-            body = body.decode("utf-8", errors="ignore")
-        elif hasattr(body, "read"):
-            body = "<file-like object>"  # Don't try to read it to avoid consuming the stream
+        parts += [("-H", f"{k}: {v}")]
+
+    body: Optional[str] = None
+    if request.content is not None:
+        body = request.content.decode("utf-8", errors="ignore")
         if len(body) > 1000:
-            body = body[:1000] + " ... [truncated]"
+            body = f"{body[:1000]} ... [truncated]"
+    elif request.stream is not None:
+        body = "<streaming body>"
+    if body is not None:
         parts += [("-d", body.replace("\n", ""))]
 
     parts += [(None, request.url)]
@@ -582,9 +745,9 @@ def _curlify(request: requests.PreparedRequest) -> str:
     flat_parts = []
     for k, v in parts:
         if k:
-            flat_parts.append(quote(k))
+            flat_parts.append(quote(str(k)))
         if v:
-            flat_parts.append(quote(v))
+            flat_parts.append(quote(str(v)))
 
     return " ".join(flat_parts)