huggingface-hub 0.12.0rc0__py3-none-any.whl → 0.13.0rc0__py3-none-any.whl

huggingface_hub/file_download.py

@@ -19,9 +19,10 @@ from urllib.parse import quote, urlparse
 
 import requests
 from filelock import FileLock
-from huggingface_hub import constants
 from requests.exceptions import ConnectTimeout, ProxyError
 
+from huggingface_hub import constants
+
 from . import __version__  # noqa: F401 # for backward compatibility
 from .constants import (
     DEFAULT_REVISION,
@@ -36,38 +37,41 @@ from .constants import (
     REPO_TYPES,
     REPO_TYPES_URL_PREFIXES,
 )
-from .utils import get_fastai_version  # noqa: F401 # for backward compatibility
-from .utils import get_fastcore_version  # noqa: F401 # for backward compatibility
-from .utils import get_graphviz_version  # noqa: F401 # for backward compatibility
-from .utils import get_jinja_version  # noqa: F401 # for backward compatibility
-from .utils import get_pydot_version  # noqa: F401 # for backward compatibility
-from .utils import get_tf_version  # noqa: F401 # for backward compatibility
-from .utils import get_torch_version  # noqa: F401 # for backward compatibility
-from .utils import is_fastai_available  # noqa: F401 # for backward compatibility
-from .utils import is_fastcore_available  # noqa: F401 # for backward compatibility
-from .utils import is_graphviz_available  # noqa: F401 # for backward compatibility
-from .utils import is_jinja_available  # noqa: F401 # for backward compatibility
-from .utils import is_pydot_available  # noqa: F401 # for backward compatibility
-from .utils import is_tf_available  # noqa: F401 # for backward compatibility
-from .utils import is_torch_available  # noqa: F401 # for backward compatibility
 from .utils import (
     EntryNotFoundError,
     LocalEntryNotFoundError,
     SoftTemporaryDirectory,
     build_hf_headers,
+    get_fastai_version,  # noqa: F401 # for backward compatibility
+    get_fastcore_version,  # noqa: F401 # for backward compatibility
+    get_graphviz_version,  # noqa: F401 # for backward compatibility
+    get_jinja_version,  # noqa: F401 # for backward compatibility
+    get_pydot_version,  # noqa: F401 # for backward compatibility
+    get_tf_version,  # noqa: F401 # for backward compatibility
+    get_torch_version,  # noqa: F401 # for backward compatibility
     hf_raise_for_status,
     http_backoff,
+    is_fastai_available,  # noqa: F401 # for backward compatibility
+    is_fastcore_available,  # noqa: F401 # for backward compatibility
+    is_graphviz_available,  # noqa: F401 # for backward compatibility
+    is_jinja_available,  # noqa: F401 # for backward compatibility
+    is_pydot_available,  # noqa: F401 # for backward compatibility
+    is_tf_available,  # noqa: F401 # for backward compatibility
+    is_torch_available,  # noqa: F401 # for backward compatibility
     logging,
     tqdm,
     validate_hf_hub_args,
 )
 from .utils._headers import _http_user_agent
 from .utils._runtime import _PY_VERSION  # noqa: F401 # for backward compatibility
-from .utils._typing import HTTP_METHOD_T
+from .utils._typing import HTTP_METHOD_T, Literal
 
 
 logger = logging.get_logger(__name__)
 
+# Regex to get filename from a "Content-Disposition" header for CDN-served files
+HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P<filename>.*?)";')
+
 
 _are_symlinks_supported_in_dir: Dict[str, bool] = {}
 
@@ -185,8 +189,8 @@ def hf_hub_url(
         subfolder (`str`, *optional*):
             An optional value corresponding to a folder inside the repo.
         repo_type (`str`, *optional*):
-            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
-            `None` or `"model"` if uploading to a model. Default is `None`.
+            Set to `"dataset"` or `"space"` if downloading from a dataset or space,
+            `None` or `"model"` if downloading from a model. Default is `None`.
         revision (`str`, *optional*):
             An optional Git revision id which can be a branch name, a tag, or a
             commit hash.
@@ -347,9 +351,7 @@ def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None):
     HF_HUB_OFFLINE is True."""
     if constants.HF_HUB_OFFLINE:
         raise OfflineModeIsEnabled(
-            "Offline mode is enabled."
-            if msg is None
-            else "Offline mode is enabled. " + str(msg)
+            "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg)
         )
 
 
@@ -515,16 +517,22 @@ def http_get(
 
     displayed_name = url
     content_disposition = r.headers.get("Content-Disposition")
-    if content_disposition is not None and "filename=" in content_disposition:
-        # Means file is on CDN
-        displayed_name = content_disposition.split("filename=")[-1]
+    if content_disposition is not None:
+        match = HEADER_FILENAME_PATTERN.search(content_disposition)
+        if match is not None:
+            # Means file is on CDN
+            displayed_name = match.groupdict()["filename"]
+
+    # Truncate filename if too long to display
+    if len(displayed_name) > 22:
+        displayed_name = f"(…){displayed_name[-20:]}"
 
     progress = tqdm(
         unit="B",
         unit_scale=True,
         total=total,
         initial=resume_size,
-        desc=f"Downloading (…){displayed_name[-20:]}",
+        desc=f"Downloading {displayed_name}",
         disable=bool(logger.getEffectiveLevel() == logging.NOTSET),
     )
     for chunk in r.iter_content(chunk_size=10 * 1024 * 1024):
@@ -627,8 +635,10 @@ def cached_download(
     """
     if not legacy_cache_layout:
         warnings.warn(
-            "`cached_download` is the legacy way to download files from the HF hub,"
-            " please consider upgrading to `hf_hub_download`",
+            (
+                "`cached_download` is the legacy way to download files from the HF hub,"
+                " please consider upgrading to `hf_hub_download`"
+            ),
             FutureWarning,
         )
 
@@ -666,8 +676,7 @@ def cached_download(
             # If we don't have any of those, raise an error.
             if etag is None:
                 raise OSError(
-                    "Distant resource does not have an ETag, we won't be able to"
-                    " reliably ensure reproducibility."
+                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
                 )
             # In case of a redirect, save an extra redirect on the request.get call,
             # and ensure we download the exact atomic version even if it changed
@@ -675,6 +684,7 @@ def cached_download(
             # Useful for lfs blobs that are stored on a CDN.
             if 300 <= r.status_code <= 399:
                 url_to_download = r.headers["Location"]
+                headers.pop("authorization", None)
         except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
             # Actually raise for those subclasses of ConnectionError
             raise
@@ -687,9 +697,7 @@ def cached_download(
             # etag is None
             pass
 
-    filename = (
-        force_filename if force_filename is not None else url_to_filename(url, etag)
-    )
+    filename = force_filename if force_filename is not None else url_to_filename(url, etag)
 
     # get cache path to put the file
     cache_path = os.path.join(cache_dir, filename)
@@ -702,16 +710,10 @@ def cached_download(
         else:
             matching_files = [
                 file
-                for file in fnmatch.filter(
-                    os.listdir(cache_dir), filename.split(".")[0] + ".*"
-                )
+                for file in fnmatch.filter(os.listdir(cache_dir), filename.split(".")[0] + ".*")
                 if not file.endswith(".json") and not file.endswith(".lock")
             ]
-            if (
-                len(matching_files) > 0
-                and not force_download
-                and force_filename is None
-            ):
+            if len(matching_files) > 0 and not force_download and force_filename is None:
                 return os.path.join(cache_dir, matching_files[-1])
             else:
                 # If files cannot be found and local_files_only=True,
@@ -843,11 +845,19 @@ def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None
     except OSError:
         pass
 
-    cache_dir = os.path.dirname(os.path.commonpath([src, dst]))
-    if are_symlinks_supported(cache_dir=cache_dir):
-        relative_src = os.path.relpath(src, start=os.path.dirname(dst))
+    try:
+        _support_symlinks = are_symlinks_supported(
+            os.path.dirname(os.path.commonpath([os.path.realpath(src), os.path.realpath(dst)]))
+        )
+    except PermissionError:
+        # Permission error means src and dst are not in the same volume (e.g. destination path has been provided
+        # by the user via `local_dir`. Let's test symlink support there)
+        _support_symlinks = are_symlinks_supported(os.path.dirname(dst))
+
+    if _support_symlinks:
+        logger.info(f"Creating pointer from {src} to {dst}")
         try:
-            os.symlink(relative_src, dst)
+            os.symlink(src, dst)
         except FileExistsError:
             if os.path.islink(dst) and os.path.realpath(dst) == os.path.realpath(src):
                 # `dst` already exists and is a symlink to the `src` blob. It is most
@@ -860,14 +870,14 @@ def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None
                 # blob file. Raise exception.
                 raise
     elif new_blob:
+        logger.info(f"Symlink not supported. Moving file from {src} to {dst}")
         os.replace(src, dst)
     else:
+        logger.info(f"Symlink not supported. Copying file from {src} to {dst}")
         shutil.copyfile(src, dst)
 
 
-def _cache_commit_hash_for_specific_revision(
-    storage_folder: str, revision: str, commit_hash: str
-) -> None:
+def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None:
     """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash.
 
     Does nothing if `revision` is already a proper `commit_hash` or reference is already cached.
@@ -905,6 +915,8 @@ def hf_hub_download(
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
     cache_dir: Union[str, Path, None] = None,
+    local_dir: Union[str, Path, None] = None,
+    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
     user_agent: Union[Dict, str, None] = None,
     force_download: bool = False,
     force_filename: Optional[str] = None,
@@ -927,6 +939,21 @@ def hf_hub_download(
           that have been resolved at that particular commit. Each filename is a symlink to the blob
           at that particular commit.
 
+    If `local_dir` is provided, the file structure from the repo will be replicated in this location. You can configure
+    how you want to move those files:
+      - If `local_dir_use_symlinks="auto"` (default), files are downloaded and stored in the cache directory as blob
+        files. Small files (<5MB) are duplicated in `local_dir` while a symlink is created for bigger files. The goal
+        is to be able to manually edit and save small files without corrupting the cache while saving disk space for
+        binary files. The 5MB threshold can be configured with the `HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD`
+        environment variable.
+      - If `local_dir_use_symlinks=True`, files are downloaded, stored in the cache directory and symlinked in `local_dir`.
+        This is optimal in term of disk usage but files must not be manually edited.
+      - If `local_dir_use_symlinks=False` and the blob files exist in the cache directory, they are duplicated in the
+        local dir. This means disk usage is not optimized.
+      - Finally, if `local_dir_use_symlinks=False` and the blob files do not exist in the cache directory, then the
+        files are downloaded and directly placed under `local_dir`. This means if you need to download them again later,
+        they will be re-downloaded entirely.
+
     ```
     [  96]  .
     └── [ 160]  models--julien-c--EsperBERTo-small
@@ -953,8 +980,8 @@ def hf_hub_download(
         subfolder (`str`, *optional*):
             An optional value corresponding to a folder inside the model repo.
         repo_type (`str`, *optional*):
-            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
-            `None` or `"model"` if uploading to a model. Default is `None`.
+            Set to `"dataset"` or `"space"` if downloading from a dataset or space,
+            `None` or `"model"` if downloading from a model. Default is `None`.
         revision (`str`, *optional*):
             An optional Git revision id which can be a branch name, a tag, or a
             commit hash.
@@ -964,6 +991,14 @@ def hf_hub_download(
             The version of the library.
         cache_dir (`str`, `Path`, *optional*):
             Path to the folder where cached files are stored.
+        local_dir (`str` or `Path`, *optional*):
+            If provided, the downloaded file will be placed under this directory, either as a symlink (default) or
+            a regular file (see description for more details).
+        local_dir_use_symlinks (`"auto"` or `bool`, defaults to `"auto"`):
+            To be used with `local_dir`. If set to "auto", the cache directory will be used and the file will be either
+            duplicated or symlinked to the local directory depending on its size. It set to `True`, a symlink will be
+            created, no matter the file size. If set to `False`, the file will either be duplicated from cache (if
+            already exists) or downloaded from the Hub and not cached. See description for more details.
         user_agent (`dict`, `str`, *optional*):
             The user-agent info in the form of a dictionary or a string.
         force_download (`bool`, *optional*, defaults to `False`):
@@ -1018,8 +1053,10 @@ def hf_hub_download(
     """
     if force_filename is not None:
         warnings.warn(
-            "The `force_filename` parameter is deprecated as a new caching system, "
-            "which keeps the filenames as they are on the Hub, is now in place.",
+            (
+                "The `force_filename` parameter is deprecated as a new caching system, "
+                "which keeps the filenames as they are on the Hub, is now in place."
+            ),
             FutureWarning,
         )
         legacy_cache_layout = True
@@ -1055,6 +1092,8 @@ def hf_hub_download(
         revision = DEFAULT_REVISION
     if isinstance(cache_dir, Path):
         cache_dir = str(cache_dir)
+    if isinstance(local_dir, Path):
+        local_dir = str(local_dir)
 
     if subfolder == "":
         subfolder = None
@@ -1065,14 +1104,9 @@ def hf_hub_download(
     if repo_type is None:
         repo_type = "model"
     if repo_type not in REPO_TYPES:
-        raise ValueError(
-            f"Invalid repo type: {repo_type}. Accepted repo types are:"
-            f" {str(REPO_TYPES)}"
-        )
+        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
 
-    storage_folder = os.path.join(
-        cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)
-    )
+    storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
     os.makedirs(storage_folder, exist_ok=True)
 
     # cross platform transcription of filename, to be used as a local file path.
@@ -1081,10 +1115,10 @@ def hf_hub_download(
     # if user provides a commit_hash and they already have the file on disk,
     # shortcut everything.
     if REGEX_COMMIT_HASH.match(revision):
-        pointer_path = os.path.join(
-            storage_folder, "snapshots", revision, relative_filename
-        )
+        pointer_path = os.path.join(storage_folder, "snapshots", revision, relative_filename)
         if os.path.exists(pointer_path):
+            if local_dir is not None:
+                return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
             return pointer_path
 
     url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision)
@@ -1110,30 +1144,18 @@ def hf_hub_download(
                 )
             except EntryNotFoundError as http_error:
                 # Cache the non-existence of the file and raise
-                commit_hash = http_error.response.headers.get(
-                    HUGGINGFACE_HEADER_X_REPO_COMMIT
-                )
+                commit_hash = http_error.response.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT)
                 if commit_hash is not None and not legacy_cache_layout:
-                    no_exist_file_path = (
-                        Path(storage_folder)
-                        / ".no_exist"
-                        / commit_hash
-                        / relative_filename
-                    )
+                    no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
                     no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
                     no_exist_file_path.touch()
-                    _cache_commit_hash_for_specific_revision(
-                        storage_folder, revision, commit_hash
-                    )
+                    _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
                 raise
 
             # Commit hash must exist
             commit_hash = metadata.commit_hash
             if commit_hash is None:
-                raise OSError(
-                    "Distant resource does not seem to be on huggingface.co (missing"
-                    " commit header)."
-                )
+                raise OSError("Distant resource does not seem to be on huggingface.co (missing commit header).")
 
             # Etag must exist
             etag = metadata.etag
@@ -1142,8 +1164,7 @@ def hf_hub_download(
             # If we don't have any of those, raise an error.
             if etag is None:
                 raise OSError(
-                    "Distant resource does not have an ETag, we won't be able to"
-                    " reliably ensure reproducibility."
+                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
                 )
 
             # In case of a redirect, save an extra redirect on the request.get call,
@@ -1174,24 +1195,30 @@ def hf_hub_download(
         # In those cases, we cannot force download.
         if force_download:
             raise ValueError(
-                "We have no connection or you passed local_files_only, so"
-                " force_download is not an accepted option."
+                "We have no connection or you passed local_files_only, so force_download is not an accepted option."
             )
+
+        # Try to get "commit_hash" from "revision"
+        commit_hash = None
         if REGEX_COMMIT_HASH.match(revision):
             commit_hash = revision
         else:
             ref_path = os.path.join(storage_folder, "refs", revision)
-            with open(ref_path) as f:
-                commit_hash = f.read()
-
-        pointer_path = os.path.join(
-            storage_folder, "snapshots", commit_hash, relative_filename
-        )
-        if os.path.exists(pointer_path):
-            return pointer_path
+            if os.path.isfile(ref_path):
+                with open(ref_path) as f:
+                    commit_hash = f.read()
+
+        # Return pointer file if exists
+        if commit_hash is not None:
+            pointer_path = os.path.join(storage_folder, "snapshots", commit_hash, relative_filename)
+            if os.path.exists(pointer_path):
+                if local_dir is not None:
+                    return _to_local_dir(
+                        pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks
+                    )
+                return pointer_path
 
-        # If we couldn't find an appropriate file on disk,
-        # raise an error.
+        # If we couldn't find an appropriate file on disk, raise an error.
         # If files cannot be found and local_files_only=True,
         # the models might've been found if local_files_only=False
         # Notify the user about that
@@ -1212,9 +1239,7 @@ def hf_hub_download(
     assert etag is not None, "etag must have been retrieved from server"
     assert commit_hash is not None, "commit_hash must have been retrieved from server"
     blob_path = os.path.join(storage_folder, "blobs", etag)
-    pointer_path = os.path.join(
-        storage_folder, "snapshots", commit_hash, relative_filename
-    )
+    pointer_path = os.path.join(storage_folder, "snapshots", commit_hash, relative_filename)
 
     os.makedirs(os.path.dirname(blob_path), exist_ok=True)
     os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
@@ -1224,13 +1249,17 @@ def hf_hub_download(
     _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
 
     if os.path.exists(pointer_path) and not force_download:
+        if local_dir is not None:
+            return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
         return pointer_path
 
     if os.path.exists(blob_path) and not force_download:
         # we have the blob already, but not the pointer
-        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
-        _create_relative_symlink(blob_path, pointer_path, new_blob=False)
-        return pointer_path
+        if local_dir is not None:  # to local dir
+            return _to_local_dir(blob_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
+        else:  # or in snapshot cache
+            _create_relative_symlink(blob_path, pointer_path, new_blob=False)
+            return pointer_path
 
     # Prevent parallel downloads of the same file with a lock.
     lock_path = blob_path + ".lock"
@@ -1281,11 +1310,31 @@ def hf_hub_download(
                 headers=headers,
             )
 
-        logger.info("storing %s in cache at %s", url, blob_path)
-        _chmod_and_replace(temp_file.name, blob_path)
-
-        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
-        _create_relative_symlink(blob_path, pointer_path, new_blob=True)
+        if local_dir is None:
+            logger.info(f"Storing {url} in cache at {blob_path}")
+            _chmod_and_replace(temp_file.name, blob_path)
+            _create_relative_symlink(blob_path, pointer_path, new_blob=True)
+        else:
+            local_dir_filepath = os.path.join(local_dir, relative_filename)
+            os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
+
+            # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
+            # In both cases, blob file is cached.
+            is_big_file = os.stat(temp_file.name).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
+            if local_dir_use_symlinks is True or (local_dir_use_symlinks == "auto" and is_big_file):
+                logger.info(f"Storing {url} in cache at {blob_path}")
+                _chmod_and_replace(temp_file.name, blob_path)
+                logger.info("Create symlink to local dir")
+                _create_relative_symlink(blob_path, local_dir_filepath, new_blob=False)
+            elif local_dir_use_symlinks == "auto" and not is_big_file:
+                logger.info(f"Storing {url} in cache at {blob_path}")
+                _chmod_and_replace(temp_file.name, blob_path)
+                logger.info("Duplicate in local dir (small file and use_symlink set to 'auto')")
+                shutil.copyfile(blob_path, local_dir_filepath)
+            else:
+                logger.info(f"Storing {url} in local_dir at {local_dir_filepath} (not cached).")
+                _chmod_and_replace(temp_file.name, local_dir_filepath)
+            pointer_path = local_dir_filepath  # for return value
 
     try:
         os.remove(lock_path)
@@ -1327,16 +1376,30 @@ def try_to_load_from_cache(
             - The exact path to the cached file if it's found in the cache
             - A special value `_CACHED_NO_EXIST` if the file does not exist at the given commit hash and this fact was
               cached.
+
+    Example:
+
+    ```python
+    from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST
+
+    filepath = try_to_load_from_cache()
+    if isinstance(filepath, str):
+        # file exists and is cached
+        ...
+    elif filepath is _CACHED_NO_EXIST:
+        # non-existence of file is cached
+        ...
+    else:
+        # file is not cached
+        ...
+    ```
     """
     if revision is None:
         revision = "main"
     if repo_type is None:
         repo_type = "model"
     if repo_type not in REPO_TYPES:
-        raise ValueError(
-            f"Invalid repo type: {repo_type}. Accepted repo types are:"
-            f" {str(REPO_TYPES)}"
-        )
+        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
     if cache_dir is None:
         cache_dir = HUGGINGFACE_HUB_CACHE
 
@@ -1345,25 +1408,32 @@ def try_to_load_from_cache(
     if not os.path.isdir(repo_cache):
         # No cache for this model
         return None
-    for subfolder in ["refs", "snapshots"]:
-        if not os.path.isdir(os.path.join(repo_cache, subfolder)):
-            return None
 
-    # Resolve refs (for instance to convert main to the associated commit sha)
-    cached_refs = os.listdir(os.path.join(repo_cache, "refs"))
-    if revision in cached_refs:
-        with open(os.path.join(repo_cache, "refs", revision)) as f:
-            revision = f.read()
+    refs_dir = os.path.join(repo_cache, "refs")
+    snapshots_dir = os.path.join(repo_cache, "snapshots")
+    no_exist_dir = os.path.join(repo_cache, ".no_exist")
 
-    if os.path.isfile(os.path.join(repo_cache, ".no_exist", revision, filename)):
+    # Resolve refs (for instance to convert main to the associated commit sha)
+    if os.path.isdir(refs_dir):
+        revision_file = os.path.join(refs_dir, revision)
+        if os.path.isfile(revision_file):
+            with open(revision_file) as f:
+                revision = f.read()
+
+    # Check if file is cached as "no_exist"
+    if os.path.isfile(os.path.join(no_exist_dir, revision, filename)):
         return _CACHED_NO_EXIST
 
-    cached_shas = os.listdir(os.path.join(repo_cache, "snapshots"))
+    # Check if revision folder exists
+    if not os.path.exists(snapshots_dir):
+        return None
+    cached_shas = os.listdir(snapshots_dir)
     if revision not in cached_shas:
         # No cache for this revision and we won't try to return a random revision
         return None
 
-    cached_file = os.path.join(repo_cache, "snapshots", revision, filename)
+    # Check if file exists in cache
+    cached_file = os.path.join(snapshots_dir, revision, filename)
     return cached_file if os.path.isfile(cached_file) else None
 
 
@@ -1422,10 +1492,7 @@ def get_hf_file_metadata(
         # Do not use directly `url`, as `_request_wrapper` might have followed relative
         # redirects.
         location=r.headers.get("Location") or r.request.url,  # type: ignore
-        size=_int_or_none(
-            r.headers.get(HUGGINGFACE_HEADER_X_LINKED_SIZE)
-            or r.headers.get("Content-Length")
-        ),
+        size=_int_or_none(r.headers.get(HUGGINGFACE_HEADER_X_LINKED_SIZE) or r.headers.get("Content-Length")),
     )
 
 
@@ -1459,3 +1526,25 @@ def _chmod_and_replace(src: str, dst: str) -> None:
         tmp_file.unlink()
 
     os.replace(src, dst)
+
+
+def _to_local_dir(
+    path: str, local_dir: str, relative_filename: str, use_symlinks: Union[bool, Literal["auto"]]
+) -> str:
+    """Place a file in a local dir (different than cache_dir).
+
+    Either symlink to blob file in cache or duplicate file depending on `use_symlinks` and file size.
+    """
+    local_dir_filepath = os.path.join(local_dir, relative_filename)
+    os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
+    real_blob_path = os.path.realpath(path)
+
+    # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
+    if use_symlinks == "auto":
+        use_symlinks = os.stat(real_blob_path).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
+
+    if use_symlinks:
+        _create_relative_symlink(real_blob_path, local_dir_filepath, new_blob=False)
+    else:
+        shutil.copyfile(real_blob_path, local_dir_filepath)
+    return local_dir_filepath