huggingface-hub 0.12.1__py3-none-any.whl → 0.13.0rc0__py3-none-any.whl

huggingface_hub/fastai_utils.py

@@ -144,16 +144,11 @@ def _check_fastai_fastcore_pyproject_versions(
     # If the package is specified but not the version (e.g. "fastai" instead of "fastai=2.4"), the default versions are the highest.
     fastai_packages = [pck for pck in package_versions if pck.startswith("fastai")]
     if len(fastai_packages) == 0:
-        logger.warning(
-            "The repository does not have a fastai version specified in the"
-            " `pyproject.toml`."
-        )
+        logger.warning("The repository does not have a fastai version specified in the `pyproject.toml`.")
     # fastai_version is an empty string if not specified
     else:
         fastai_version = str(fastai_packages[0]).partition("=")[2]
-        if fastai_version != "" and version.Version(fastai_version) < version.Version(
-            fastai_min_version
-        ):
+        if fastai_version != "" and version.Version(fastai_version) < version.Version(fastai_min_version):
             raise ImportError(
                 "`from_pretrained_fastai` requires"
                 f" fastai>={fastai_min_version} version but the model to load uses"
@@ -162,16 +157,11 @@ def _check_fastai_fastcore_pyproject_versions(
 
     fastcore_packages = [pck for pck in package_versions if pck.startswith("fastcore")]
     if len(fastcore_packages) == 0:
-        logger.warning(
-            "The repository does not have a fastcore version specified in the"
-            " `pyproject.toml`."
-        )
+        logger.warning("The repository does not have a fastcore version specified in the `pyproject.toml`.")
     # fastcore_version is an empty string if not specified
     else:
         fastcore_version = str(fastcore_packages[0]).partition("=")[2]
-        if fastcore_version != "" and version.Version(
-            fastcore_version
-        ) < version.Version(fastcore_min_version):
+        if fastcore_version != "" and version.Version(fastcore_version) < version.Version(fastcore_min_version):
             raise ImportError(
                 "`from_pretrained_fastai` requires"
                 f" fastcore>={fastcore_min_version} version, but you are using fastcore"
@@ -281,9 +271,7 @@ def _save_pretrained_fastai(
     # if the user provides config then we update it with the fastai and fastcore versions in CONFIG_TEMPLATE.
     if config is not None:
         if not isinstance(config, dict):
-            raise RuntimeError(
-                f"Provided config should be a dict. Got: '{type(config)}'"
-            )
+            raise RuntimeError(f"Provided config should be a dict. Got: '{type(config)}'")
         path = os.path.join(save_directory, CONFIG_NAME)
         with open(path, "w") as f:
             json.dump(config, f)
@@ -365,13 +353,15 @@ def push_to_hub_fastai(
     create_pr: Optional[bool] = None,
     allow_patterns: Optional[Union[List[str], str]] = None,
     ignore_patterns: Optional[Union[List[str], str]] = None,
+    delete_patterns: Optional[Union[List[str], str]] = None,
     api_endpoint: Optional[str] = None,
 ):
     """
     Upload learner checkpoint files to the Hub.
 
-    Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be
-    pushed to the hub. See [`upload_folder`] reference for more details.
+    Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use
+    `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more
+    details.
 
     Args:
         learner (`Learner`):
@@ -399,6 +389,9 @@ def push_to_hub_fastai(
             If provided, only files matching at least one pattern are pushed.
         ignore_patterns (`List[str]` or `str`, *optional*):
             If provided, files matching any of the patterns are not pushed.
+        delete_patterns (`List[str]` or `str`, *optional*):
+            If provided, remote files matching any of the patterns will be deleted from the repo.
+
     Returns:
         The url of the commit of your model in the given repository.
 
@@ -413,9 +406,7 @@ def push_to_hub_fastai(
     """
     _check_fastai_fastcore_versions()
     api = HfApi(endpoint=api_endpoint)
-    api.create_repo(
-        repo_id=repo_id, repo_type="model", token=token, private=private, exist_ok=True
-    )
+    repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id
 
     # Push the files to the repo in a single commit
     with SoftTemporaryDirectory() as tmp:
@@ -423,7 +414,6 @@ def push_to_hub_fastai(
         _save_pretrained_fastai(learner, saved_path, config=config)
         return api.upload_folder(
             repo_id=repo_id,
-            repo_type="model",
             token=token,
             folder_path=saved_path,
             commit_message=commit_message,
@@ -431,4 +421,5 @@ def push_to_hub_fastai(
             create_pr=create_pr,
             allow_patterns=allow_patterns,
             ignore_patterns=ignore_patterns,
+            delete_patterns=delete_patterns,
         )

huggingface_hub/file_download.py

@@ -19,9 +19,10 @@ from urllib.parse import quote, urlparse
 
 import requests
 from filelock import FileLock
-from huggingface_hub import constants
 from requests.exceptions import ConnectTimeout, ProxyError
 
+from huggingface_hub import constants
+
 from . import __version__  # noqa: F401 # for backward compatibility
 from .constants import (
     DEFAULT_REVISION,
@@ -36,38 +37,41 @@ from .constants import (
     REPO_TYPES,
     REPO_TYPES_URL_PREFIXES,
 )
-from .utils import get_fastai_version  # noqa: F401 # for backward compatibility
-from .utils import get_fastcore_version  # noqa: F401 # for backward compatibility
-from .utils import get_graphviz_version  # noqa: F401 # for backward compatibility
-from .utils import get_jinja_version  # noqa: F401 # for backward compatibility
-from .utils import get_pydot_version  # noqa: F401 # for backward compatibility
-from .utils import get_tf_version  # noqa: F401 # for backward compatibility
-from .utils import get_torch_version  # noqa: F401 # for backward compatibility
-from .utils import is_fastai_available  # noqa: F401 # for backward compatibility
-from .utils import is_fastcore_available  # noqa: F401 # for backward compatibility
-from .utils import is_graphviz_available  # noqa: F401 # for backward compatibility
-from .utils import is_jinja_available  # noqa: F401 # for backward compatibility
-from .utils import is_pydot_available  # noqa: F401 # for backward compatibility
-from .utils import is_tf_available  # noqa: F401 # for backward compatibility
-from .utils import is_torch_available  # noqa: F401 # for backward compatibility
 from .utils import (
     EntryNotFoundError,
     LocalEntryNotFoundError,
     SoftTemporaryDirectory,
     build_hf_headers,
+    get_fastai_version,  # noqa: F401 # for backward compatibility
+    get_fastcore_version,  # noqa: F401 # for backward compatibility
+    get_graphviz_version,  # noqa: F401 # for backward compatibility
+    get_jinja_version,  # noqa: F401 # for backward compatibility
+    get_pydot_version,  # noqa: F401 # for backward compatibility
+    get_tf_version,  # noqa: F401 # for backward compatibility
+    get_torch_version,  # noqa: F401 # for backward compatibility
     hf_raise_for_status,
     http_backoff,
+    is_fastai_available,  # noqa: F401 # for backward compatibility
+    is_fastcore_available,  # noqa: F401 # for backward compatibility
+    is_graphviz_available,  # noqa: F401 # for backward compatibility
+    is_jinja_available,  # noqa: F401 # for backward compatibility
+    is_pydot_available,  # noqa: F401 # for backward compatibility
+    is_tf_available,  # noqa: F401 # for backward compatibility
+    is_torch_available,  # noqa: F401 # for backward compatibility
     logging,
     tqdm,
     validate_hf_hub_args,
 )
 from .utils._headers import _http_user_agent
 from .utils._runtime import _PY_VERSION  # noqa: F401 # for backward compatibility
-from .utils._typing import HTTP_METHOD_T
+from .utils._typing import HTTP_METHOD_T, Literal
 
 
 logger = logging.get_logger(__name__)
 
+# Regex to get filename from a "Content-Disposition" header for CDN-served files
+HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P<filename>.*?)";')
+
 
 _are_symlinks_supported_in_dir: Dict[str, bool] = {}
 
@@ -185,8 +189,8 @@ def hf_hub_url(
         subfolder (`str`, *optional*):
             An optional value corresponding to a folder inside the repo.
         repo_type (`str`, *optional*):
-            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
-            `None` or `"model"` if uploading to a model. Default is `None`.
+            Set to `"dataset"` or `"space"` if downloading from a dataset or space,
+            `None` or `"model"` if downloading from a model. Default is `None`.
         revision (`str`, *optional*):
             An optional Git revision id which can be a branch name, a tag, or a
             commit hash.
@@ -347,9 +351,7 @@ def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None):
     HF_HUB_OFFLINE is True."""
     if constants.HF_HUB_OFFLINE:
         raise OfflineModeIsEnabled(
-            "Offline mode is enabled."
-            if msg is None
-            else "Offline mode is enabled. " + str(msg)
+            "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg)
         )
 
 
@@ -515,16 +517,22 @@ def http_get(
 
     displayed_name = url
     content_disposition = r.headers.get("Content-Disposition")
-    if content_disposition is not None and "filename=" in content_disposition:
-        # Means file is on CDN
-        displayed_name = content_disposition.split("filename=")[-1]
+    if content_disposition is not None:
+        match = HEADER_FILENAME_PATTERN.search(content_disposition)
+        if match is not None:
+            # Means file is on CDN
+            displayed_name = match.groupdict()["filename"]
+
+    # Truncate filename if too long to display
+    if len(displayed_name) > 22:
+        displayed_name = f"(…){displayed_name[-20:]}"
 
     progress = tqdm(
         unit="B",
         unit_scale=True,
         total=total,
         initial=resume_size,
-        desc=f"Downloading (…){displayed_name[-20:]}",
+        desc=f"Downloading {displayed_name}",
         disable=bool(logger.getEffectiveLevel() == logging.NOTSET),
     )
     for chunk in r.iter_content(chunk_size=10 * 1024 * 1024):
@@ -627,8 +635,10 @@ def cached_download(
     """
     if not legacy_cache_layout:
         warnings.warn(
-            "`cached_download` is the legacy way to download files from the HF hub,"
-            " please consider upgrading to `hf_hub_download`",
+            (
+                "`cached_download` is the legacy way to download files from the HF hub,"
+                " please consider upgrading to `hf_hub_download`"
+            ),
             FutureWarning,
         )
 
@@ -666,8 +676,7 @@ def cached_download(
             # If we don't have any of those, raise an error.
             if etag is None:
                 raise OSError(
-                    "Distant resource does not have an ETag, we won't be able to"
-                    " reliably ensure reproducibility."
+                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
                 )
             # In case of a redirect, save an extra redirect on the request.get call,
             # and ensure we download the exact atomic version even if it changed
@@ -688,9 +697,7 @@ def cached_download(
             # etag is None
             pass
 
-    filename = (
-        force_filename if force_filename is not None else url_to_filename(url, etag)
-    )
+    filename = force_filename if force_filename is not None else url_to_filename(url, etag)
 
     # get cache path to put the file
     cache_path = os.path.join(cache_dir, filename)
@@ -703,16 +710,10 @@ def cached_download(
         else:
             matching_files = [
                 file
-                for file in fnmatch.filter(
-                    os.listdir(cache_dir), filename.split(".")[0] + ".*"
-                )
+                for file in fnmatch.filter(os.listdir(cache_dir), filename.split(".")[0] + ".*")
                 if not file.endswith(".json") and not file.endswith(".lock")
             ]
-            if (
-                len(matching_files) > 0
-                and not force_download
-                and force_filename is None
-            ):
+            if len(matching_files) > 0 and not force_download and force_filename is None:
                 return os.path.join(cache_dir, matching_files[-1])
             else:
                 # If files cannot be found and local_files_only=True,
@@ -844,11 +845,19 @@ def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None
     except OSError:
         pass
 
-    cache_dir = os.path.dirname(os.path.commonpath([src, dst]))
-    if are_symlinks_supported(cache_dir=cache_dir):
-        relative_src = os.path.relpath(src, start=os.path.dirname(dst))
+    try:
+        _support_symlinks = are_symlinks_supported(
+            os.path.dirname(os.path.commonpath([os.path.realpath(src), os.path.realpath(dst)]))
+        )
+    except PermissionError:
+        # Permission error means src and dst are not in the same volume (e.g. destination path has been provided
+        # by the user via `local_dir`. Let's test symlink support there)
+        _support_symlinks = are_symlinks_supported(os.path.dirname(dst))
+
+    if _support_symlinks:
+        logger.info(f"Creating pointer from {src} to {dst}")
         try:
-            os.symlink(relative_src, dst)
+            os.symlink(src, dst)
         except FileExistsError:
             if os.path.islink(dst) and os.path.realpath(dst) == os.path.realpath(src):
                 # `dst` already exists and is a symlink to the `src` blob. It is most
@@ -861,14 +870,14 @@ def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None
                 # blob file. Raise exception.
                 raise
     elif new_blob:
+        logger.info(f"Symlink not supported. Moving file from {src} to {dst}")
         os.replace(src, dst)
     else:
+        logger.info(f"Symlink not supported. Copying file from {src} to {dst}")
         shutil.copyfile(src, dst)
 
 
-def _cache_commit_hash_for_specific_revision(
-    storage_folder: str, revision: str, commit_hash: str
-) -> None:
+def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None:
     """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash.
 
     Does nothing if `revision` is already a proper `commit_hash` or reference is already cached.
@@ -906,6 +915,8 @@ def hf_hub_download(
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
     cache_dir: Union[str, Path, None] = None,
+    local_dir: Union[str, Path, None] = None,
+    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
     user_agent: Union[Dict, str, None] = None,
     force_download: bool = False,
     force_filename: Optional[str] = None,
@@ -928,6 +939,21 @@ def hf_hub_download(
           that have been resolved at that particular commit. Each filename is a symlink to the blob
           at that particular commit.
 
+    If `local_dir` is provided, the file structure from the repo will be replicated in this location. You can configure
+    how you want to move those files:
+      - If `local_dir_use_symlinks="auto"` (default), files are downloaded and stored in the cache directory as blob
+        files. Small files (<5MB) are duplicated in `local_dir` while a symlink is created for bigger files. The goal
+        is to be able to manually edit and save small files without corrupting the cache while saving disk space for
+        binary files. The 5MB threshold can be configured with the `HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD`
+        environment variable.
+      - If `local_dir_use_symlinks=True`, files are downloaded, stored in the cache directory and symlinked in `local_dir`.
+        This is optimal in term of disk usage but files must not be manually edited.
+      - If `local_dir_use_symlinks=False` and the blob files exist in the cache directory, they are duplicated in the
+        local dir. This means disk usage is not optimized.
+      - Finally, if `local_dir_use_symlinks=False` and the blob files do not exist in the cache directory, then the
+        files are downloaded and directly placed under `local_dir`. This means if you need to download them again later,
+        they will be re-downloaded entirely.
+
     ```
     [  96]  .
     └── [ 160]  models--julien-c--EsperBERTo-small
@@ -954,8 +980,8 @@ def hf_hub_download(
         subfolder (`str`, *optional*):
             An optional value corresponding to a folder inside the model repo.
         repo_type (`str`, *optional*):
-            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
-            `None` or `"model"` if uploading to a model. Default is `None`.
+            Set to `"dataset"` or `"space"` if downloading from a dataset or space,
+            `None` or `"model"` if downloading from a model. Default is `None`.
         revision (`str`, *optional*):
             An optional Git revision id which can be a branch name, a tag, or a
             commit hash.
@@ -965,6 +991,14 @@ def hf_hub_download(
             The version of the library.
         cache_dir (`str`, `Path`, *optional*):
             Path to the folder where cached files are stored.
+        local_dir (`str` or `Path`, *optional*):
+            If provided, the downloaded file will be placed under this directory, either as a symlink (default) or
+            a regular file (see description for more details).
+        local_dir_use_symlinks (`"auto"` or `bool`, defaults to `"auto"`):
+            To be used with `local_dir`. If set to "auto", the cache directory will be used and the file will be either
+            duplicated or symlinked to the local directory depending on its size. It set to `True`, a symlink will be
+            created, no matter the file size. If set to `False`, the file will either be duplicated from cache (if
+            already exists) or downloaded from the Hub and not cached. See description for more details.
         user_agent (`dict`, `str`, *optional*):
             The user-agent info in the form of a dictionary or a string.
         force_download (`bool`, *optional*, defaults to `False`):
@@ -1019,8 +1053,10 @@ def hf_hub_download(
     """
     if force_filename is not None:
         warnings.warn(
-            "The `force_filename` parameter is deprecated as a new caching system, "
-            "which keeps the filenames as they are on the Hub, is now in place.",
+            (
+                "The `force_filename` parameter is deprecated as a new caching system, "
+                "which keeps the filenames as they are on the Hub, is now in place."
+            ),
             FutureWarning,
         )
         legacy_cache_layout = True
@@ -1056,6 +1092,8 @@ def hf_hub_download(
         revision = DEFAULT_REVISION
     if isinstance(cache_dir, Path):
         cache_dir = str(cache_dir)
+    if isinstance(local_dir, Path):
+        local_dir = str(local_dir)
 
     if subfolder == "":
         subfolder = None
@@ -1066,14 +1104,9 @@ def hf_hub_download(
     if repo_type is None:
         repo_type = "model"
     if repo_type not in REPO_TYPES:
-        raise ValueError(
-            f"Invalid repo type: {repo_type}. Accepted repo types are:"
-            f" {str(REPO_TYPES)}"
-        )
+        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
 
-    storage_folder = os.path.join(
-        cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)
-    )
+    storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
     os.makedirs(storage_folder, exist_ok=True)
 
     # cross platform transcription of filename, to be used as a local file path.
@@ -1082,10 +1115,10 @@ def hf_hub_download(
     # if user provides a commit_hash and they already have the file on disk,
     # shortcut everything.
     if REGEX_COMMIT_HASH.match(revision):
-        pointer_path = os.path.join(
-            storage_folder, "snapshots", revision, relative_filename
-        )
+        pointer_path = os.path.join(storage_folder, "snapshots", revision, relative_filename)
         if os.path.exists(pointer_path):
+            if local_dir is not None:
+                return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
             return pointer_path
 
     url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision)
@@ -1111,30 +1144,18 @@ def hf_hub_download(
                 )
             except EntryNotFoundError as http_error:
                 # Cache the non-existence of the file and raise
-                commit_hash = http_error.response.headers.get(
-                    HUGGINGFACE_HEADER_X_REPO_COMMIT
-                )
+                commit_hash = http_error.response.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT)
                 if commit_hash is not None and not legacy_cache_layout:
-                    no_exist_file_path = (
-                        Path(storage_folder)
-                        / ".no_exist"
-                        / commit_hash
-                        / relative_filename
-                    )
+                    no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
                     no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
                     no_exist_file_path.touch()
-                    _cache_commit_hash_for_specific_revision(
-                        storage_folder, revision, commit_hash
-                    )
+                    _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
                 raise
 
             # Commit hash must exist
             commit_hash = metadata.commit_hash
             if commit_hash is None:
-                raise OSError(
-                    "Distant resource does not seem to be on huggingface.co (missing"
-                    " commit header)."
-                )
+                raise OSError("Distant resource does not seem to be on huggingface.co (missing commit header).")
 
             # Etag must exist
             etag = metadata.etag
@@ -1143,8 +1164,7 @@ def hf_hub_download(
             # If we don't have any of those, raise an error.
             if etag is None:
                 raise OSError(
-                    "Distant resource does not have an ETag, we won't be able to"
-                    " reliably ensure reproducibility."
+                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
                 )
 
             # In case of a redirect, save an extra redirect on the request.get call,
@@ -1175,8 +1195,7 @@ def hf_hub_download(
         # In those cases, we cannot force download.
         if force_download:
             raise ValueError(
-                "We have no connection or you passed local_files_only, so"
-                " force_download is not an accepted option."
+                "We have no connection or you passed local_files_only, so force_download is not an accepted option."
             )
 
         # Try to get "commit_hash" from "revision"
@@ -1191,14 +1210,15 @@ def hf_hub_download(
 
         # Return pointer file if exists
         if commit_hash is not None:
-            pointer_path = os.path.join(
-                storage_folder, "snapshots", commit_hash, relative_filename
-            )
+            pointer_path = os.path.join(storage_folder, "snapshots", commit_hash, relative_filename)
             if os.path.exists(pointer_path):
+                if local_dir is not None:
+                    return _to_local_dir(
+                        pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks
+                    )
                 return pointer_path
 
-        # If we couldn't find an appropriate file on disk,
-        # raise an error.
+        # If we couldn't find an appropriate file on disk, raise an error.
         # If files cannot be found and local_files_only=True,
         # the models might've been found if local_files_only=False
         # Notify the user about that
@@ -1219,9 +1239,7 @@ def hf_hub_download(
     assert etag is not None, "etag must have been retrieved from server"
     assert commit_hash is not None, "commit_hash must have been retrieved from server"
     blob_path = os.path.join(storage_folder, "blobs", etag)
-    pointer_path = os.path.join(
-        storage_folder, "snapshots", commit_hash, relative_filename
-    )
+    pointer_path = os.path.join(storage_folder, "snapshots", commit_hash, relative_filename)
 
     os.makedirs(os.path.dirname(blob_path), exist_ok=True)
     os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
@@ -1231,13 +1249,17 @@ def hf_hub_download(
     _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
 
     if os.path.exists(pointer_path) and not force_download:
+        if local_dir is not None:
+            return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
         return pointer_path
 
     if os.path.exists(blob_path) and not force_download:
         # we have the blob already, but not the pointer
-        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
-        _create_relative_symlink(blob_path, pointer_path, new_blob=False)
-        return pointer_path
+        if local_dir is not None:  # to local dir
+            return _to_local_dir(blob_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
+        else:  # or in snapshot cache
+            _create_relative_symlink(blob_path, pointer_path, new_blob=False)
+            return pointer_path
 
     # Prevent parallel downloads of the same file with a lock.
     lock_path = blob_path + ".lock"
@@ -1288,11 +1310,31 @@ def hf_hub_download(
                 headers=headers,
             )
 
-        logger.info("storing %s in cache at %s", url, blob_path)
-        _chmod_and_replace(temp_file.name, blob_path)
-
-        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
-        _create_relative_symlink(blob_path, pointer_path, new_blob=True)
+        if local_dir is None:
+            logger.info(f"Storing {url} in cache at {blob_path}")
+            _chmod_and_replace(temp_file.name, blob_path)
+            _create_relative_symlink(blob_path, pointer_path, new_blob=True)
+        else:
+            local_dir_filepath = os.path.join(local_dir, relative_filename)
+            os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
+
+            # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
+            # In both cases, blob file is cached.
+            is_big_file = os.stat(temp_file.name).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
+            if local_dir_use_symlinks is True or (local_dir_use_symlinks == "auto" and is_big_file):
+                logger.info(f"Storing {url} in cache at {blob_path}")
+                _chmod_and_replace(temp_file.name, blob_path)
+                logger.info("Create symlink to local dir")
+                _create_relative_symlink(blob_path, local_dir_filepath, new_blob=False)
+            elif local_dir_use_symlinks == "auto" and not is_big_file:
+                logger.info(f"Storing {url} in cache at {blob_path}")
+                _chmod_and_replace(temp_file.name, blob_path)
+                logger.info("Duplicate in local dir (small file and use_symlink set to 'auto')")
+                shutil.copyfile(blob_path, local_dir_filepath)
+            else:
+                logger.info(f"Storing {url} in local_dir at {local_dir_filepath} (not cached).")
+                _chmod_and_replace(temp_file.name, local_dir_filepath)
+            pointer_path = local_dir_filepath  # for return value
 
     try:
         os.remove(lock_path)
@@ -1357,10 +1399,7 @@ def try_to_load_from_cache(
     if repo_type is None:
         repo_type = "model"
     if repo_type not in REPO_TYPES:
-        raise ValueError(
-            f"Invalid repo type: {repo_type}. Accepted repo types are:"
-            f" {str(REPO_TYPES)}"
-        )
+        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
     if cache_dir is None:
         cache_dir = HUGGINGFACE_HUB_CACHE
 
@@ -1376,9 +1415,9 @@ def try_to_load_from_cache(
 
     # Resolve refs (for instance to convert main to the associated commit sha)
     if os.path.isdir(refs_dir):
-        cached_refs = os.listdir(refs_dir)
-        if revision in cached_refs:
-            with open(os.path.join(refs_dir, revision)) as f:
+        revision_file = os.path.join(refs_dir, revision)
+        if os.path.isfile(revision_file):
+            with open(revision_file) as f:
                 revision = f.read()
 
     # Check if file is cached as "no_exist"
@@ -1453,10 +1492,7 @@ def get_hf_file_metadata(
         # Do not use directly `url`, as `_request_wrapper` might have followed relative
         # redirects.
         location=r.headers.get("Location") or r.request.url,  # type: ignore
-        size=_int_or_none(
-            r.headers.get(HUGGINGFACE_HEADER_X_LINKED_SIZE)
-            or r.headers.get("Content-Length")
-        ),
+        size=_int_or_none(r.headers.get(HUGGINGFACE_HEADER_X_LINKED_SIZE) or r.headers.get("Content-Length")),
     )
 
 
@@ -1490,3 +1526,25 @@ def _chmod_and_replace(src: str, dst: str) -> None:
         tmp_file.unlink()
 
     os.replace(src, dst)
+
+
+def _to_local_dir(
+    path: str, local_dir: str, relative_filename: str, use_symlinks: Union[bool, Literal["auto"]]
+) -> str:
+    """Place a file in a local dir (different than cache_dir).
+
+    Either symlink to blob file in cache or duplicate file depending on `use_symlinks` and file size.
+    """
+    local_dir_filepath = os.path.join(local_dir, relative_filename)
+    os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
+    real_blob_path = os.path.realpath(path)
+
+    # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
+    if use_symlinks == "auto":
+        use_symlinks = os.stat(real_blob_path).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
+
+    if use_symlinks:
+        _create_relative_symlink(real_blob_path, local_dir_filepath, new_blob=False)
+    else:
+        shutil.copyfile(real_blob_path, local_dir_filepath)
+    return local_dir_filepath