huggingface-hub 0.22.2__py3-none-any.whl → 0.23.0__py3-none-any.whl

huggingface_hub/file_download.py

@@ -2,28 +2,27 @@ import copy
 import errno
 import fnmatch
 import inspect
-import io
 import json
 import os
 import re
 import shutil
 import stat
-import tempfile
 import time
 import uuid
 import warnings
-from contextlib import contextmanager
 from dataclasses import dataclass
-from functools import partial
 from pathlib import Path
-from typing import Any, BinaryIO, Dict, Generator, Literal, Optional, Tuple, Union
+from typing import Any, BinaryIO, Dict, Literal, NoReturn, Optional, Tuple, Union
 from urllib.parse import quote, urlparse
 
 import requests
 
-from huggingface_hub import constants
-
 from . import __version__  # noqa: F401 # for backward compatibility
+from ._local_folder import (
+    get_local_download_paths,
+    read_download_metadata,
+    write_download_metadata,
+)
 from .constants import (
     DEFAULT_ETAG_TIMEOUT,
     DEFAULT_REQUEST_TIMEOUT,
@@ -80,13 +79,23 @@ from .utils import (
 from .utils._runtime import _PY_VERSION  # noqa: F401 # for backward compatibility
 from .utils._typing import HTTP_METHOD_T
 from .utils.insecure_hashlib import sha256
+from .utils.sha import sha_fileobj
 
 
 logger = logging.get_logger(__name__)
 
+# Return value when trying to load a file from cache but the file does not exist in the distant repo.
+_CACHED_NO_EXIST = object()
+_CACHED_NO_EXIST_T = Any
+
 # Regex to get filename from a "Content-Disposition" header for CDN-served files
 HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P<filename>.*?)";')
 
+# Regex to check if the revision IS directly a commit_hash
+REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$")
+
+# Regex to check if the file etag IS a valid sha256
+REGEX_SHA256 = re.compile(r"^[0-9a-f]{64}$")
 
 _are_symlinks_supported_in_dir: Dict[str, bool] = {}
 
@@ -150,12 +159,6 @@ def are_symlinks_supported(cache_dir: Union[str, Path, None] = None) -> bool:
     return _are_symlinks_supported_in_dir[cache_dir]
 
 
-# Return value when trying to load a file from cache but the file does not exist in the distant repo.
-_CACHED_NO_EXIST = object()
-_CACHED_NO_EXIST_T = Any
-REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$")
-
-
 @dataclass(frozen=True)
 class HfFileMetadata:
     """Data structure containing information about a file versioned on the Hub.
@@ -478,9 +481,9 @@ def http_get(
 
     consistency_error_message = (
         f"Consistency check failed: file should be of size {expected_size} but has size"
-        f" {{actual_size}} ({displayed_filename}).\nWe are sorry for the inconvenience. Please retry download and"
-        " pass `force_download=True, resume_download=False` as argument.\nIf the issue persists, please let us"
-        " know by opening an issue on https://github.com/huggingface/huggingface_hub."
+        f" {{actual_size}} ({displayed_filename}).\nWe are sorry for the inconvenience. Please retry"
+        " with `force_download=True`.\nIf the issue persists, please let us know by opening an issue "
+        "on https://github.com/huggingface/huggingface_hub."
     )
 
     # Stream file to buffer
@@ -495,6 +498,7 @@ def http_get(
             disable=True if (logger.getEffectiveLevel() == logging.NOTSET) else None,
             # ^ set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
             # see https://github.com/huggingface/huggingface_hub/pull/2000
+            name="huggingface_hub.http_get",
         )
 
     if hf_transfer and total is not None and total > 5 * DOWNLOAD_CHUNK_SIZE:
@@ -582,7 +586,7 @@ def cached_download(
     force_filename: Optional[str] = None,
     proxies: Optional[Dict] = None,
     etag_timeout: float = DEFAULT_ETAG_TIMEOUT,
-    resume_download: bool = False,
+    resume_download: Optional[bool] = None,
     token: Union[bool, str, None] = None,
     local_files_only: bool = False,
     legacy_cache_layout: bool = False,
@@ -619,8 +623,6 @@ def cached_download(
         etag_timeout (`float`, *optional* defaults to `10`):
             When fetching ETag, how many seconds to wait for the server to send
             data before giving up which is passed to `requests.request`.
-        resume_download (`bool`, *optional*, defaults to `False`):
-            If `True`, resume a previously interrupted download.
         token (`bool`, `str`, *optional*):
             A token to be used for the download.
                 - If `True`, the token is read from the HuggingFace config
@@ -671,6 +673,13 @@ def cached_download(
             " 'hf_hub_download'",
             FutureWarning,
         )
+    if resume_download is not None:
+        warnings.warn(
+            "`resume_download` is deprecated and will be removed in version 1.0.0. "
+            "Downloads always resume when possible. "
+            "If you want to force a new download, use `force_download=True`.",
+            FutureWarning,
+        )
 
     if cache_dir is None:
         cache_dir = HF_HUB_CACHE
@@ -786,46 +795,16 @@ def cached_download(
         cache_path = "\\\\?\\" + os.path.abspath(cache_path)
 
     with WeakFileLock(lock_path):
-        # If the download just completed while the lock was activated.
-        if os.path.exists(cache_path) and not force_download:
-            # Even if returning early like here, the lock will be released.
-            return cache_path
-
-        if resume_download:
-            incomplete_path = cache_path + ".incomplete"
-
-            @contextmanager
-            def _resumable_file_manager() -> Generator[io.BufferedWriter, None, None]:
-                with open(incomplete_path, "ab") as f:
-                    yield f
-
-            temp_file_manager = _resumable_file_manager
-            if os.path.exists(incomplete_path):
-                resume_size = os.stat(incomplete_path).st_size
-            else:
-                resume_size = 0
-        else:
-            temp_file_manager = partial(  # type: ignore
-                tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False
-            )
-            resume_size = 0
-
-        # Download to temporary file, then copy to cache dir once finished.
-        # Otherwise you get corrupt cache entries if the download gets interrupted.
-        with temp_file_manager() as temp_file:
-            logger.info("downloading %s to %s", url, temp_file.name)
-
-            http_get(
-                url_to_download,
-                temp_file,
-                proxies=proxies,
-                resume_size=resume_size,
-                headers=headers,
-                expected_size=expected_size,
-            )
-
-        logger.info("storing %s in cache at %s", url, cache_path)
-        _chmod_and_replace(temp_file.name, cache_path)
+        _download_to_tmp_and_move(
+            incomplete_path=Path(cache_path + ".incomplete"),
+            destination_path=Path(cache_path),
+            url_to_download=url_to_download,
+            proxies=proxies,
+            headers=headers,
+            expected_size=expected_size,
+            filename=filename,
+            force_download=force_download,
+        )
 
         if force_filename is None:
             logger.info("creating metadata file for %s", cache_path)
@@ -1022,18 +1001,19 @@ def hf_hub_download(
     library_version: Optional[str] = None,
     cache_dir: Union[str, Path, None] = None,
     local_dir: Union[str, Path, None] = None,
-    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
     user_agent: Union[Dict, str, None] = None,
     force_download: bool = False,
-    force_filename: Optional[str] = None,
     proxies: Optional[Dict] = None,
     etag_timeout: float = DEFAULT_ETAG_TIMEOUT,
-    resume_download: bool = False,
     token: Union[bool, str, None] = None,
     local_files_only: bool = False,
     headers: Optional[Dict[str, str]] = None,
-    legacy_cache_layout: bool = False,
     endpoint: Optional[str] = None,
+    # Deprecated args
+    legacy_cache_layout: bool = False,
+    resume_download: Optional[bool] = None,
+    force_filename: Optional[str] = None,
+    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
 ) -> str:
     """Download a given file if it's not already present in the local cache.
 
@@ -1047,21 +1027,6 @@ def hf_hub_download(
           that have been resolved at that particular commit. Each filename is a symlink to the blob
           at that particular commit.
 
-    If `local_dir` is provided, the file structure from the repo will be replicated in this location. You can configure
-    how you want to move those files:
-      - If `local_dir_use_symlinks="auto"` (default), files are downloaded and stored in the cache directory as blob
-        files. Small files (<5MB) are duplicated in `local_dir` while a symlink is created for bigger files. The goal
-        is to be able to manually edit and save small files without corrupting the cache while saving disk space for
-        binary files. The 5MB threshold can be configured with the `HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD`
-        environment variable.
-      - If `local_dir_use_symlinks=True`, files are downloaded, stored in the cache directory and symlinked in `local_dir`.
-        This is optimal in term of disk usage but files must not be manually edited.
-      - If `local_dir_use_symlinks=False` and the blob files exist in the cache directory, they are duplicated in the
-        local dir. This means disk usage is not optimized.
-      - Finally, if `local_dir_use_symlinks=False` and the blob files do not exist in the cache directory, then the
-        files are downloaded and directly placed under `local_dir`. This means if you need to download them again later,
-        they will be re-downloaded entirely.
-
     ```
     [  96]  .
     └── [ 160]  models--julien-c--EsperBERTo-small
@@ -1080,6 +1045,11 @@ def hf_hub_download(
                 └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
     ```
 
+    If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this
+    option, the `cache_dir` will not be used and a `.huggingface/` folder will be created at the root of `local_dir`
+    to store some metadata related to the downloaded files. While this mechanism is not as robust as the main
+    cache-system, it's optimized for regularly pulling the latest version of a repository.
+
     Args:
         repo_id (`str`):
             A user or an organization name and a repo name separated by a `/`.
@@ -1100,13 +1070,7 @@ def hf_hub_download(
         cache_dir (`str`, `Path`, *optional*):
             Path to the folder where cached files are stored.
         local_dir (`str` or `Path`, *optional*):
-            If provided, the downloaded file will be placed under this directory, either as a symlink (default) or
-            a regular file (see description for more details).
-        local_dir_use_symlinks (`"auto"` or `bool`, defaults to `"auto"`):
-            To be used with `local_dir`. If set to "auto", the cache directory will be used and the file will be either
-            duplicated or symlinked to the local directory depending on its size. It set to `True`, a symlink will be
-            created, no matter the file size. If set to `False`, the file will either be duplicated from cache (if
-            already exists) or downloaded from the Hub and not cached. See description for more details.
+            If provided, the downloaded file will be placed under this directory.
         user_agent (`dict`, `str`, *optional*):
             The user-agent info in the form of a dictionary or a string.
         force_download (`bool`, *optional*, defaults to `False`):
@@ -1118,8 +1082,6 @@ def hf_hub_download(
         etag_timeout (`float`, *optional*, defaults to `10`):
             When fetching ETag, how many seconds to wait for the server to send
             data before giving up which is passed to `requests.request`.
-        resume_download (`bool`, *optional*, defaults to `False`):
-            If `True`, resume a previously interrupted download.
         token (`str`, `bool`, *optional*):
             A token to be used for the download.
                 - If `True`, the token is read from the HuggingFace config
@@ -1136,30 +1098,24 @@ def hf_hub_download(
             more powerful.
 
     Returns:
-        Local path (string) of file or if networking is off, last version of
-        file cached on disk.
-
-    <Tip>
-
-    Raises the following errors:
+        `str`: Local path of file or if networking is off, last version of file cached on disk.
 
+    Raises:
         - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
-          if `token=True` and the token cannot be found.
+        if `token=True` and the token cannot be found.
         - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
-          if ETag cannot be determined.
+        if ETag cannot be determined.
         - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-          if some parameter value is invalid
+        if some parameter value is invalid
         - [`~utils.RepositoryNotFoundError`]
-          If the repository to download from cannot be found. This may be because it doesn't exist,
-          or because it is set to `private` and you do not have access.
+        If the repository to download from cannot be found. This may be because it doesn't exist,
+        or because it is set to `private` and you do not have access.
         - [`~utils.RevisionNotFoundError`]
-          If the revision to download from cannot be found.
+        If the revision to download from cannot be found.
         - [`~utils.EntryNotFoundError`]
-          If the file to download cannot be found.
+        If the file to download cannot be found.
         - [`~utils.LocalEntryNotFoundError`]
-          If network is disabled or unavailable and file is not found in cache.
-
-    </Tip>
+        If network is disabled or unavailable and file is not found in cache.
     """
     if HF_HUB_ETAG_TIMEOUT != DEFAULT_ETAG_TIMEOUT:
         # Respect environment variable above user value
@@ -1172,6 +1128,13 @@ def hf_hub_download(
             FutureWarning,
         )
         legacy_cache_layout = True
+    if resume_download is not None:
+        warnings.warn(
+            "`resume_download` is deprecated and will be removed in version 1.0.0. "
+            "Downloads always resume when possible. "
+            "If you want to force a new download, use `force_download=True`.",
+            FutureWarning,
+        )
 
     if legacy_cache_layout:
         url = hf_hub_url(
@@ -1193,7 +1156,6 @@ def hf_hub_download(
             force_filename=force_filename,
             proxies=proxies,
             etag_timeout=etag_timeout,
-            resume_download=resume_download,
             token=token,
             local_files_only=local_files_only,
             legacy_cache_layout=legacy_cache_layout,
@@ -1207,7 +1169,6 @@ def hf_hub_download(
         cache_dir = str(cache_dir)
     if isinstance(local_dir, Path):
         local_dir = str(local_dir)
-    locks_dir = os.path.join(cache_dir, ".locks")
 
     if subfolder == "":
         subfolder = None
@@ -1220,6 +1181,85 @@ def hf_hub_download(
     if repo_type not in REPO_TYPES:
         raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
 
+    headers = build_hf_headers(
+        token=token,
+        library_name=library_name,
+        library_version=library_version,
+        user_agent=user_agent,
+        headers=headers,
+    )
+
+    if local_dir is not None:
+        if local_dir_use_symlinks != "auto":
+            warnings.warn(
+                "`local_dir_use_symlinks` parameter is deprecated and will be ignored. "
+                "The process to download files to a local folder has been updated and do "
+                "not rely on symlinks anymore. You only need to pass a destination folder "
+                "as`local_dir`.\n"
+                "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder."
+            )
+
+        return _hf_hub_download_to_local_dir(
+            # Destination
+            local_dir=local_dir,
+            # File info
+            repo_id=repo_id,
+            repo_type=repo_type,
+            filename=filename,
+            revision=revision,
+            # HTTP info
+            proxies=proxies,
+            etag_timeout=etag_timeout,
+            headers=headers,
+            endpoint=endpoint,
+            # Additional options
+            cache_dir=cache_dir,
+            force_download=force_download,
+            local_files_only=local_files_only,
+        )
+    else:
+        return _hf_hub_download_to_cache_dir(
+            # Destination
+            cache_dir=cache_dir,
+            # File info
+            repo_id=repo_id,
+            filename=filename,
+            repo_type=repo_type,
+            revision=revision,
+            # HTTP info
+            headers=headers,
+            proxies=proxies,
+            etag_timeout=etag_timeout,
+            endpoint=endpoint,
+            # Additional options
+            local_files_only=local_files_only,
+            force_download=force_download,
+        )
+
+
+def _hf_hub_download_to_cache_dir(
+    *,
+    # Destination
+    cache_dir: str,
+    # File info
+    repo_id: str,
+    filename: str,
+    repo_type: str,
+    revision: str,
+    # HTTP info
+    headers: Dict[str, str],
+    proxies: Optional[Dict],
+    etag_timeout: float,
+    endpoint: Optional[str],
+    # Additional options
+    local_files_only: bool,
+    force_download: bool,
+) -> str:
+    """Download a given file to a cache folder, if not already present.
+
+    Method should not be called directly. Please use `hf_hub_download` instead.
+    """
+    locks_dir = os.path.join(cache_dir, ".locks")
     storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
 
     # cross platform transcription of filename, to be used as a local file path.
@@ -1231,207 +1271,82 @@ def hf_hub_download(
                 " owner to rename this file."
             )
 
-    # if user provides a commit_hash and they already have the file on disk,
-    # shortcut everything.
+    # if user provides a commit_hash and they already have the file on disk, shortcut everything.
     if REGEX_COMMIT_HASH.match(revision):
         pointer_path = _get_pointer_path(storage_folder, revision, relative_filename)
-        if os.path.exists(pointer_path):
-            if local_dir is not None:
-                return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
+        if os.path.exists(pointer_path) and not force_download:
             return pointer_path
 
-    url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision, endpoint=endpoint)
-
-    headers = build_hf_headers(
-        token=token,
-        library_name=library_name,
-        library_version=library_version,
-        user_agent=user_agent,
+    # Try to get metadata (etag, commit_hash, url, size) from the server.
+    # If we can't, a HEAD request error is returned.
+    (url_to_download, etag, commit_hash, expected_size, head_call_error) = _get_metadata_or_catch_error(
+        repo_id=repo_id,
+        filename=filename,
+        repo_type=repo_type,
+        revision=revision,
+        endpoint=endpoint,
+        proxies=proxies,
+        etag_timeout=etag_timeout,
         headers=headers,
+        local_files_only=local_files_only,
+        storage_folder=storage_folder,
+        relative_filename=relative_filename,
     )
 
-    url_to_download = url
-    etag = None
-    commit_hash = None
-    expected_size = None
-    head_call_error: Optional[Exception] = None
-    if not local_files_only:
-        try:
-            try:
-                metadata = get_hf_file_metadata(
-                    url=url,
-                    token=token,
-                    proxies=proxies,
-                    timeout=etag_timeout,
-                    library_name=library_name,
-                    library_version=library_version,
-                    user_agent=user_agent,
-                )
-            except EntryNotFoundError as http_error:
-                # Cache the non-existence of the file and raise
-                commit_hash = http_error.response.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT)
-                if commit_hash is not None and not legacy_cache_layout:
-                    no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
-                    no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
-                    no_exist_file_path.touch()
-                    _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
-                raise
-
-            # Commit hash must exist
-            commit_hash = metadata.commit_hash
-            if commit_hash is None:
-                raise FileMetadataError(
-                    "Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue"
-                    " prevents you from downloading resources from https://huggingface.co. Please check your firewall"
-                    " and proxy settings and make sure your SSL certificates are updated."
-                )
-
-            # Etag must exist
-            etag = metadata.etag
-            # We favor a custom header indicating the etag of the linked resource, and
-            # we fallback to the regular etag header.
-            # If we don't have any of those, raise an error.
-            if etag is None:
-                raise FileMetadataError(
-                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
-                )
-
-            # Expected (uncompressed) size
-            expected_size = metadata.size
-
-            # In case of a redirect, save an extra redirect on the request.get call,
-            # and ensure we download the exact atomic version even if it changed
-            # between the HEAD and the GET (unlikely, but hey).
-            #
-            # If url domain is different => we are downloading from a CDN => url is signed => don't send auth
-            # If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth
-            if metadata.location != url:
-                url_to_download = metadata.location
-                if urlparse(url).netloc != urlparse(url_to_download).netloc:
-                    # Remove authorization header when downloading a LFS blob
-                    headers.pop("authorization", None)
-        except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
-            # Actually raise for those subclasses of ConnectionError
-            raise
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.Timeout,
-            OfflineModeIsEnabled,
-        ) as error:
-            # Otherwise, our Internet connection is down.
-            # etag is None
-            head_call_error = error
-            pass
-        except (RevisionNotFoundError, EntryNotFoundError):
-            # The repo was found but the revision or entry doesn't exist on the Hub (never existed or got deleted)
-            raise
-        except requests.HTTPError as error:
-            # Multiple reasons for an http error:
-            # - Repository is private and invalid/missing token sent
-            # - Repository is gated and invalid/missing token sent
-            # - Hub is down (error 500 or 504)
-            # => let's switch to 'local_files_only=True' to check if the files are already cached.
-            #    (if it's not the case, the error will be re-raised)
-            head_call_error = error
-            pass
-        except FileMetadataError as error:
-            # Multiple reasons for a FileMetadataError:
-            # - Wrong network configuration (proxy, firewall, SSL certificates)
-            # - Inconsistency on the Hub
-            # => let's switch to 'local_files_only=True' to check if the files are already cached.
-            #    (if it's not the case, the error will be re-raised)
-            head_call_error = error
-            pass
-
-    assert (
-        local_files_only or etag is not None or head_call_error is not None
-    ), "etag is empty due to uncovered problems"
-
     # etag can be None for several reasons:
     # 1. we passed local_files_only.
     # 2. we don't have a connection
-    # 3. Hub is down (HTTP 500 or 504)
+    # 3. Hub is down (HTTP 500, 503, 504)
     # 4. repo is not found -for example private or gated- and invalid/missing token sent
     # 5. Hub is blocked by a firewall or proxy is not set correctly.
     # => Try to get the last downloaded one from the specified revision.
     #
     # If the specified revision is a commit hash, look inside "snapshots".
     # If the specified revision is a branch or tag, look inside "refs".
-    if etag is None:
-        # In those cases, we cannot force download.
-        if force_download:
-            if local_files_only:
-                raise ValueError("Cannot pass 'force_download=True' and 'local_files_only=True' at the same time.")
-            elif isinstance(head_call_error, OfflineModeIsEnabled):
-                raise ValueError(
-                    "Cannot pass 'force_download=True' when offline mode is enabled."
-                ) from head_call_error
+    if head_call_error is not None:
+        # Couldn't make a HEAD call => let's try to find a local file
+        if not force_download:
+            commit_hash = None
+            if REGEX_COMMIT_HASH.match(revision):
+                commit_hash = revision
             else:
-                raise ValueError("Force download failed due to the above error.") from head_call_error
+                ref_path = os.path.join(storage_folder, "refs", revision)
+                if os.path.isfile(ref_path):
+                    with open(ref_path) as f:
+                        commit_hash = f.read()
 
-        # Try to get "commit_hash" from "revision"
-        commit_hash = None
-        if REGEX_COMMIT_HASH.match(revision):
-            commit_hash = revision
-        else:
-            ref_path = os.path.join(storage_folder, "refs", revision)
-            if os.path.isfile(ref_path):
-                with open(ref_path) as f:
-                    commit_hash = f.read()
-
-        # Return pointer file if exists
-        if commit_hash is not None:
-            pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)
-            if os.path.exists(pointer_path):
-                if local_dir is not None:
-                    return _to_local_dir(
-                        pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks
-                    )
-                return pointer_path
+            # Return pointer file if exists
+            if commit_hash is not None:
+                pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)
+                if os.path.exists(pointer_path) and not force_download:
+                    return pointer_path
 
-        # If we couldn't find an appropriate file on disk, raise an error.
-        # If files cannot be found and local_files_only=True,
-        # the models might've been found if local_files_only=False
-        # Notify the user about that
-        if local_files_only:
-            raise LocalEntryNotFoundError(
-                "Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable"
-                " hf.co look-ups and downloads online, set 'local_files_only' to False."
-            )
-        elif isinstance(head_call_error, RepositoryNotFoundError) or isinstance(head_call_error, GatedRepoError):
-            # Repo not found or gated => let's raise the actual error
-            raise head_call_error
-        else:
-            # Otherwise: most likely a connection issue or Hub downtime => let's warn the user
-            raise LocalEntryNotFoundError(
-                "An error happened while trying to locate the file on the Hub and we cannot find the requested files"
-                " in the local cache. Please check your connection and try again or make sure your Internet connection"
-                " is on."
-            ) from head_call_error
-
-    # From now on, etag and commit_hash are not None.
+        # Otherwise, raise appropriate error
+        _raise_on_head_call_error(head_call_error, force_download, local_files_only)
+
+    # From now on, etag, commit_hash, url and size are not None.
     assert etag is not None, "etag must have been retrieved from server"
     assert commit_hash is not None, "commit_hash must have been retrieved from server"
+    assert url_to_download is not None, "file location must have been retrieved from server"
+    assert expected_size is not None, "expected_size must have been retrieved from server"
     blob_path = os.path.join(storage_folder, "blobs", etag)
     pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)
 
     os.makedirs(os.path.dirname(blob_path), exist_ok=True)
     os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
+
     # if passed revision is not identical to commit_hash
     # then revision has to be a branch name or tag name.
     # In that case store a ref.
     _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
 
-    if os.path.exists(pointer_path) and not force_download:
-        if local_dir is not None:
-            return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
-        return pointer_path
+    # If file already exists, return it (except if force_download=True)
+    if not force_download:
+        if os.path.exists(pointer_path):
+            return pointer_path
 
-    if os.path.exists(blob_path) and not force_download:
-        # we have the blob already, but not the pointer
-        if local_dir is not None:  # to local dir
-            return _to_local_dir(blob_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
-        else:  # or in snapshot cache
+        if os.path.exists(blob_path):
+            # we have the blob already, but not the pointer
             _create_symlink(blob_path, pointer_path, new_blob=False)
             return pointer_path
 
@@ -1449,83 +1364,139 @@ def hf_hub_download(
 
     Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
     with WeakFileLock(lock_path):
-        # If the download just completed while the lock was activated.
-        if os.path.exists(pointer_path) and not force_download:
-            # Even if returning early like here, the lock will be released.
-            if local_dir is not None:
-                return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
-            return pointer_path
-
-        if resume_download:
-            incomplete_path = blob_path + ".incomplete"
-
-            @contextmanager
-            def _resumable_file_manager() -> Generator[io.BufferedWriter, None, None]:
-                with open(incomplete_path, "ab") as f:
-                    yield f
+        _download_to_tmp_and_move(
+            incomplete_path=Path(blob_path + ".incomplete"),
+            destination_path=Path(blob_path),
+            url_to_download=url_to_download,
+            proxies=proxies,
+            headers=headers,
+            expected_size=expected_size,
+            filename=filename,
+            force_download=force_download,
+        )
+        _create_symlink(blob_path, pointer_path, new_blob=True)
 
-            temp_file_manager = _resumable_file_manager
-            if os.path.exists(incomplete_path):
-                resume_size = os.stat(incomplete_path).st_size
-            else:
-                resume_size = 0
-        else:
-            temp_file_manager = partial(  # type: ignore
-                tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False
-            )
-            resume_size = 0
+    return pointer_path
 
-        # Download to temporary file, then copy to cache dir once finished.
-        # Otherwise you get corrupt cache entries if the download gets interrupted.
-        with temp_file_manager() as temp_file:
-            logger.info("downloading %s to %s", url, temp_file.name)
 
-            if expected_size is not None:  # might be None if HTTP header not set correctly
-                # Check tmp path
-                _check_disk_space(expected_size, os.path.dirname(temp_file.name))
+def _hf_hub_download_to_local_dir(
+    *,
+    # Destination
+    local_dir: Union[str, Path],
+    # File info
+    repo_id: str,
+    repo_type: str,
+    filename: str,
+    revision: str,
+    # HTTP info
+    proxies: Optional[Dict],
+    etag_timeout: float,
+    headers: Dict[str, str],
+    endpoint: Optional[str],
+    # Additional options
+    cache_dir: str,
+    force_download: bool,
+    local_files_only: bool,
+) -> str:
+    """Download a given file to a local folder, if not already present.
 
-                # Check destination
-                _check_disk_space(expected_size, os.path.dirname(blob_path))
-                if local_dir is not None:
-                    _check_disk_space(expected_size, local_dir)
+    Method should not be called directly. Please use `hf_hub_download` instead.
+    """
+    local_dir = Path(local_dir)
+    paths = get_local_download_paths(local_dir=local_dir, filename=filename)
+    local_metadata = read_download_metadata(local_dir=local_dir, filename=filename)
+
+    # Local file exists + metadata exists + commit_hash matches => return file
+    if (
+        not force_download
+        and REGEX_COMMIT_HASH.match(revision)
+        and paths.file_path.is_file()
+        and local_metadata is not None
+        and local_metadata.commit_hash == revision
+    ):
+        return str(paths.file_path)
+
+    # Local file doesn't exist or commit_hash doesn't match => we need the etag
+    (url_to_download, etag, commit_hash, expected_size, head_call_error) = _get_metadata_or_catch_error(
+        repo_id=repo_id,
+        filename=filename,
+        repo_type=repo_type,
+        revision=revision,
+        endpoint=endpoint,
+        proxies=proxies,
+        etag_timeout=etag_timeout,
+        headers=headers,
+        local_files_only=local_files_only,
+    )
 
-            http_get(
-                url_to_download,
-                temp_file,
-                proxies=proxies,
-                resume_size=resume_size,
-                headers=headers,
-                expected_size=expected_size,
-                displayed_filename=filename,
+    if head_call_error is not None:
+        # No HEAD call but local file exists => default to local file
+        if not force_download and paths.file_path.is_file():
+            logger.warning(
+                f"Couldn't access the Hub to check for update but local file already exists. Defaulting to existing file. (error: {head_call_error})"
             )
+            return str(paths.file_path)
+        # Otherwise => raise
+        _raise_on_head_call_error(head_call_error, force_download, local_files_only)
 
-        if local_dir is None:
-            logger.debug(f"Storing {url} in cache at {blob_path}")
-            _chmod_and_replace(temp_file.name, blob_path)
-            _create_symlink(blob_path, pointer_path, new_blob=True)
-        else:
-            local_dir_filepath = os.path.join(local_dir, relative_filename)
-            os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
-
-            # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
-            # In both cases, blob file is cached.
-            is_big_file = os.stat(temp_file.name).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
-            if local_dir_use_symlinks is True or (local_dir_use_symlinks == "auto" and is_big_file):
-                logger.debug(f"Storing {url} in cache at {blob_path}")
-                _chmod_and_replace(temp_file.name, blob_path)
-                logger.debug("Create symlink to local dir")
-                _create_symlink(blob_path, local_dir_filepath, new_blob=False)
-            elif local_dir_use_symlinks == "auto" and not is_big_file:
-                logger.debug(f"Storing {url} in cache at {blob_path}")
-                _chmod_and_replace(temp_file.name, blob_path)
-                logger.debug("Duplicate in local dir (small file and use_symlink set to 'auto')")
-                shutil.copyfile(blob_path, local_dir_filepath)
-            else:
-                logger.debug(f"Storing {url} in local_dir at {local_dir_filepath} (not cached).")
-                _chmod_and_replace(temp_file.name, local_dir_filepath)
-            pointer_path = local_dir_filepath  # for return value
+    # From now on, etag, commit_hash, url and size are not None.
+    assert etag is not None, "etag must have been retrieved from server"
+    assert commit_hash is not None, "commit_hash must have been retrieved from server"
+    assert url_to_download is not None, "file location must have been retrieved from server"
+    assert expected_size is not None, "expected_size must have been retrieved from server"
+
+    # Local file exists => check if it's up-to-date
+    if not force_download and paths.file_path.is_file():
+        # etag matches => update metadata and return file
+        if local_metadata is not None and local_metadata.etag == etag:
+            write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag)
+            return str(paths.file_path)
+
+        # metadata is outdated + etag is a sha256
+        # => means it's an LFS file (large)
+        # => let's compute local hash and compare
+        # => if match, update metadata and return file
+        if local_metadata is None and REGEX_SHA256.match(etag) is not None:
+            with open(paths.file_path, "rb") as f:
+                file_hash = sha_fileobj(f).hex()
+            if file_hash == etag:
+                write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag)
+                return str(paths.file_path)
+
+    # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache)
+
+    # If we are lucky enough, the file is already in the cache => copy it
+    if not force_download:
+        cached_path = try_to_load_from_cache(
+            repo_id=repo_id,
+            filename=filename,
+            cache_dir=cache_dir,
+            revision=commit_hash,
+            repo_type=repo_type,
+        )
+        if isinstance(cached_path, str):
+            with WeakFileLock(paths.lock_path):
+                paths.file_path.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(cached_path, paths.file_path)
+            write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag)
+            return str(paths.file_path)
+
+    # Otherwise, let's download the file!
+    with WeakFileLock(paths.lock_path):
+        paths.file_path.unlink(missing_ok=True)  # delete outdated file first
+        _download_to_tmp_and_move(
+            incomplete_path=paths.incomplete_path(etag),
+            destination_path=paths.file_path,
+            url_to_download=url_to_download,
+            proxies=proxies,
+            headers=headers,
+            expected_size=expected_size,
+            filename=filename,
+            force_download=force_download,
+        )
 
-    return pointer_path
+    write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag)
+    return str(paths.file_path)
 
 
 @validate_hf_hub_args
@@ -1696,6 +1667,233 @@ def get_hf_file_metadata(
     )
 
 
+def _get_metadata_or_catch_error(
+    *,
+    repo_id: str,
+    filename: str,
+    repo_type: str,
+    revision: str,
+    endpoint: Optional[str],
+    proxies: Optional[Dict],
+    etag_timeout: Optional[float],
+    headers: Dict[str, str],  # mutated inplace!
+    local_files_only: bool,
+    relative_filename: Optional[str] = None,  # only used to store `.no_exists` in cache
+    storage_folder: Optional[str] = None,  # only used to store `.no_exists` in cache
+) -> Union[
+    # Either an exception is caught and returned
+    Tuple[None, None, None, None, Exception],
+    # Or the metadata is returned as
+    # `(url_to_download, etag, commit_hash, expected_size, None)`
+    Tuple[str, str, str, int, None],
+]:
+    """Get metadata for a file on the Hub, safely handling network issues.
+
+    Returns either the etag, commit_hash and expected size of the file, or the error
+    raised while fetching the metadata.
+
+    NOTE: This function mutates `headers` inplace! It removes the `authorization` header
+          if the file is a LFS blob and the domain of the url is different from the
+          domain of the location (typically an S3 bucket).
+    """
+    if local_files_only:
+        return (
+            None,
+            None,
+            None,
+            None,
+            OfflineModeIsEnabled(
+                f"Cannot access file since 'local_files_only=True' as been set. (repo_id: {repo_id}, repo_type: {repo_type}, revision: {revision}, filename: {filename})"
+            ),
+        )
+
+    url = url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision, endpoint=endpoint)
+    url_to_download: str = url
+    etag: Optional[str] = None
+    commit_hash: Optional[str] = None
+    expected_size: Optional[int] = None
+    head_error_call: Optional[Exception] = None
+
+    # Try to get metadata from the server.
+    # Do not raise yet if the file is not found or not accessible.
+    if not local_files_only:
+        try:
+            try:
+                metadata = get_hf_file_metadata(url=url, proxies=proxies, timeout=etag_timeout, headers=headers)
+            except EntryNotFoundError as http_error:
+                if storage_folder is not None and relative_filename is not None:
+                    # Cache the non-existence of the file
+                    commit_hash = http_error.response.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT)
+                    if commit_hash is not None:
+                        no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
+                        no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
+                        no_exist_file_path.touch()
+                        _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
+                raise
+
+            # Commit hash must exist
+            commit_hash = metadata.commit_hash
+            if commit_hash is None:
+                raise FileMetadataError(
+                    "Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue"
+                    " prevents you from downloading resources from https://huggingface.co. Please check your firewall"
+                    " and proxy settings and make sure your SSL certificates are updated."
+                )
+
+            # Etag must exist
+            # If we don't have any of those, raise an error.
+            etag = metadata.etag
+            if etag is None:
+                raise FileMetadataError(
+                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
+                )
+
+            # Size must exist
+            expected_size = metadata.size
+            if expected_size is None:
+                raise FileMetadataError("Distant resource does not have a Content-Length.")
+
+            # In case of a redirect, save an extra redirect on the request.get call,
+            # and ensure we download the exact atomic version even if it changed
+            # between the HEAD and the GET (unlikely, but hey).
+            #
+            # If url domain is different => we are downloading from a CDN => url is signed => don't send auth
+            # If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth
+            if url != metadata.location:
+                url_to_download = metadata.location
+                if urlparse(url).netloc != urlparse(metadata.location).netloc:
+                    # Remove authorization header when downloading a LFS blob
+                    headers.pop("authorization", None)
+        except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
+            # Actually raise for those subclasses of ConnectionError
+            raise
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout,
+            OfflineModeIsEnabled,
+        ) as error:
+            # Otherwise, our Internet connection is down.
+            # etag is None
+            head_error_call = error
+        except (RevisionNotFoundError, EntryNotFoundError):
+            # The repo was found but the revision or entry doesn't exist on the Hub (never existed or got deleted)
+            raise
+        except requests.HTTPError as error:
+            # Multiple reasons for an http error:
+            # - Repository is private and invalid/missing token sent
+            # - Repository is gated and invalid/missing token sent
+            # - Hub is down (error 500 or 504)
+            # => let's switch to 'local_files_only=True' to check if the files are already cached.
+            #    (if it's not the case, the error will be re-raised)
+            head_error_call = error
+        except FileMetadataError as error:
+            # Multiple reasons for a FileMetadataError:
+            # - Wrong network configuration (proxy, firewall, SSL certificates)
+            # - Inconsistency on the Hub
+            # => let's switch to 'local_files_only=True' to check if the files are already cached.
+            #    (if it's not the case, the error will be re-raised)
+            head_error_call = error
+
+    if not (local_files_only or etag is not None or head_error_call is not None):
+        raise RuntimeError("etag is empty due to uncovered problems")
+
+    return (url_to_download, etag, commit_hash, expected_size, head_error_call)  # type: ignore [return-value]
+
+
+def _raise_on_head_call_error(head_call_error: Exception, force_download: bool, local_files_only: bool) -> NoReturn:
+    """Raise an appropriate error when the HEAD call failed and we cannot locate a local file."""
+
+    # No head call => we cannot force download.
+    if force_download:
+        if local_files_only:
+            raise ValueError("Cannot pass 'force_download=True' and 'local_files_only=True' at the same time.")
+        elif isinstance(head_call_error, OfflineModeIsEnabled):
+            raise ValueError("Cannot pass 'force_download=True' when offline mode is enabled.") from head_call_error
+        else:
+            raise ValueError("Force download failed due to the above error.") from head_call_error
+
+    # No head call + couldn't find an appropriate file on disk => raise an error.
+    if local_files_only:
+        raise LocalEntryNotFoundError(
+            "Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable"
+            " hf.co look-ups and downloads online, set 'local_files_only' to False."
+        )
+    elif isinstance(head_call_error, RepositoryNotFoundError) or isinstance(head_call_error, GatedRepoError):
+        # Repo not found or gated => let's raise the actual error
+        raise head_call_error
+    else:
+        # Otherwise: most likely a connection issue or Hub downtime => let's warn the user
+        raise LocalEntryNotFoundError(
+            "An error happened while trying to locate the file on the Hub and we cannot find the requested files"
+            " in the local cache. Please check your connection and try again or make sure your Internet connection"
+            " is on."
+        ) from head_call_error
+
+
+def _download_to_tmp_and_move(
+    incomplete_path: Path,
+    destination_path: Path,
+    url_to_download: str,
+    proxies: Optional[Dict],
+    headers: Dict[str, str],
+    expected_size: Optional[int],
+    filename: str,
+    force_download: bool,
+) -> None:
+    """Download content from a URL to a destination path.
+
+    Internal logic:
+    - return early if file is already downloaded
+    - resume download if possible (from incomplete file)
+    - do not resume download if `force_download=True` or `HF_HUB_ENABLE_HF_TRANSFER=True`
+    - check disk space before downloading
+    - download content to a temporary file
+    - set correct permissions on temporary file
+    - move the temporary file to the destination path
+
+    Both `incomplete_path` and `destination_path` must be on the same volume to avoid a local copy.
+    """
+    if destination_path.exists() and not force_download:
+        # Do nothing if already exists (except if force_download=True)
+        return
+
+    if incomplete_path.exists() and (force_download or (HF_HUB_ENABLE_HF_TRANSFER and not proxies)):
+        # By default, we will try to resume the download if possible.
+        # However, if the user has set `force_download=True` or if `hf_transfer` is enabled, then we should
+        # not resume the download => delete the incomplete file.
+        message = f"Removing incomplete file '{incomplete_path}'"
+        if force_download:
+            message += " (force_download=True)"
+        elif HF_HUB_ENABLE_HF_TRANSFER and not proxies:
+            message += " (hf_transfer=True)"
+        logger.info(message)
+        incomplete_path.unlink(missing_ok=True)
+
+    with incomplete_path.open("ab") as f:
+        resume_size = f.tell()
+        message = f"Downloading '{filename}' to '{incomplete_path}'"
+        if resume_size > 0 and expected_size is not None:
+            message += f" (resume from {resume_size}/{expected_size})"
+        logger.info(message)
+
+        if expected_size is not None:  # might be None if HTTP header not set correctly
+            # Check disk space in both tmp and destination path
+            _check_disk_space(expected_size, incomplete_path.parent)
+            _check_disk_space(expected_size, destination_path.parent)
+
+        http_get(
+            url_to_download,
+            f,
+            proxies=proxies,
+            resume_size=resume_size,
+            headers=headers,
+            expected_size=expected_size,
+        )
+
+    logger.info(f"Download complete. Moving file to {destination_path}")
+    _chmod_and_move(incomplete_path, destination_path)
+
+
 def _int_or_none(value: Optional[str]) -> Optional[int]:
     try:
         return int(value)  # type: ignore
@@ -1703,7 +1901,7 @@ def _int_or_none(value: Optional[str]) -> Optional[int]:
         return None
 
 
-def _chmod_and_replace(src: str, dst: str) -> None:
+def _chmod_and_move(src: Path, dst: Path) -> None:
     """Set correct permission before moving a blob from tmp directory to cache dir.
 
     Do not take into account the `umask` from the process as there is no convenient way
@@ -1717,15 +1915,15 @@ def _chmod_and_replace(src: str, dst: str) -> None:
     - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1215
     """
     # Get umask by creating a temporary file in the cached repo folder.
-    tmp_file = Path(dst).parent.parent / f"tmp_{uuid.uuid4()}"
+    tmp_file = dst.parent.parent / f"tmp_{uuid.uuid4()}"
     try:
         tmp_file.touch()
         cache_dir_mode = Path(tmp_file).stat().st_mode
-        os.chmod(src, stat.S_IMODE(cache_dir_mode))
+        os.chmod(str(src), stat.S_IMODE(cache_dir_mode))
     finally:
         tmp_file.unlink()
 
-    shutil.move(src, dst)
+    shutil.move(str(src), str(dst))
 
 
 def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str) -> str:
@@ -1739,32 +1937,3 @@ def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str
             f" `relative_filename='{relative_filename}'`."
         )
     return pointer_path
-
-
-def _to_local_dir(
-    path: str, local_dir: str, relative_filename: str, use_symlinks: Union[bool, Literal["auto"]]
-) -> str:
-    """Place a file in a local dir (different than cache_dir).
-
-    Either symlink to blob file in cache or duplicate file depending on `use_symlinks` and file size.
-    """
-    # Using `os.path.abspath` instead of `Path.resolve()` to avoid resolving symlinks
-    local_dir_filepath = os.path.join(local_dir, relative_filename)
-    if Path(os.path.abspath(local_dir)) not in Path(os.path.abspath(local_dir_filepath)).parents:
-        raise ValueError(
-            f"Cannot copy file '{relative_filename}' to local dir '{local_dir}': file would not be in the local"
-            " directory."
-        )
-
-    os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
-    real_blob_path = os.path.realpath(path)
-
-    # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
-    if use_symlinks == "auto":
-        use_symlinks = os.stat(real_blob_path).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
-
-    if use_symlinks:
-        _create_symlink(real_blob_path, local_dir_filepath, new_blob=False)
-    else:
-        shutil.copyfile(real_blob_path, local_dir_filepath)
-    return local_dir_filepath