huggingface-hub 0.21.4__py3-none-any.whl → 0.22.0rc0__py3-none-any.whl

huggingface_hub/file_download.py

@@ -1,4 +1,5 @@
 import copy
+import errno
 import fnmatch
 import inspect
 import io
@@ -19,7 +20,6 @@ from typing import Any, BinaryIO, Dict, Generator, Literal, Optional, Tuple, Uni
 from urllib.parse import quote, urlparse
 
 import requests
-from filelock import FileLock
 
 from huggingface_hub import constants
 
@@ -54,6 +54,7 @@ from .utils import (
     RepositoryNotFoundError,
     RevisionNotFoundError,
     SoftTemporaryDirectory,
+    WeakFileLock,
     build_hf_headers,
     get_fastai_version,  # noqa: F401 # for backward compatibility
     get_fastcore_version,  # noqa: F401 # for backward compatibility
@@ -76,8 +77,6 @@ from .utils import (
     tqdm,
     validate_hf_hub_args,
 )
-from .utils._deprecation import _deprecate_method
-from .utils._headers import _http_user_agent
 from .utils._runtime import _PY_VERSION  # noqa: F401 # for backward compatibility
 from .utils._typing import HTTP_METHOD_T
 from .utils.insecure_hashlib import sha256
@@ -347,21 +346,6 @@ def filename_to_url(
     return url, etag
 
 
-@_deprecate_method(version="0.22.0", message="Use `huggingface_hub.utils.build_hf_headers` instead.")
-def http_user_agent(
-    *,
-    library_name: Optional[str] = None,
-    library_version: Optional[str] = None,
-    user_agent: Union[Dict, str, None] = None,
-) -> str:
-    """Deprecated in favor of [`build_hf_headers`]."""
-    return _http_user_agent(
-        library_name=library_name,
-        library_version=library_version,
-        user_agent=user_agent,
-    )
-
-
 def _request_wrapper(
     method: HTTP_METHOD_T, url: str, *, follow_relative_redirects: bool = False, **params
 ) -> requests.Response:
@@ -794,7 +778,7 @@ def cached_download(
     if os.name == "nt" and len(os.path.abspath(cache_path)) > 255:
         cache_path = "\\\\?\\" + os.path.abspath(cache_path)
 
-    with FileLock(lock_path):
+    with WeakFileLock(lock_path):
         # If the download just completed while the lock was activated.
         if os.path.exists(cache_path) and not force_download:
             # Even if returning early like here, the lock will be released.
@@ -878,7 +862,7 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
 
     By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages:
     - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will
-      not brake.
+      not break.
     - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when
       changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398,
       https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228.
@@ -899,7 +883,7 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
     cache, the file is duplicated on the disk.
 
     In case symlinks are not supported, a warning message is displayed to the user once when loading `huggingface_hub`.
-    The warning message can be disable with the `DISABLE_SYMLINKS_WARNING` environment variable.
+    The warning message can be disabled with the `DISABLE_SYMLINKS_WARNING` environment variable.
     """
     try:
         os.remove(dst)
@@ -930,6 +914,12 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
         # Permission error means src and dst are not in the same volume (e.g. destination path has been provided
         # by the user via `local_dir`. Let's test symlink support there)
         _support_symlinks = are_symlinks_supported(abs_dst_folder)
+    except OSError as e:
+        # OS error (errno=30) means that the commonpath is readonly on Linux/MacOS.
+        if e.errno == errno.EROFS:
+            _support_symlinks = are_symlinks_supported(abs_dst_folder)
+        else:
+            raise
 
     # Symlinks are supported => let's create a symlink.
     if _support_symlinks:
@@ -1034,6 +1024,7 @@ def hf_hub_download(
     resume_download: bool = False,
     token: Union[bool, str, None] = None,
     local_files_only: bool = False,
+    headers: Optional[Dict[str, str]] = None,
     legacy_cache_layout: bool = False,
     endpoint: Optional[str] = None,
 ) -> str:
@@ -1130,6 +1121,8 @@ def hf_hub_download(
         local_files_only (`bool`, *optional*, defaults to `False`):
             If `True`, avoid downloading the file and return the path to the
             local cached file if it exists.
+        headers (`dict`, *optional*):
+            Additional headers to be sent with the request.
         legacy_cache_layout (`bool`, *optional*, defaults to `False`):
             If `True`, uses the legacy file cache layout i.e. just call [`hf_hub_url`]
             then `cached_download`. This is deprecated as the new cache layout is
@@ -1221,7 +1214,6 @@ def hf_hub_download(
         raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
 
     storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
-    os.makedirs(storage_folder, exist_ok=True)
 
     # cross platform transcription of filename, to be used as a local file path.
     relative_filename = os.path.join(*filename.split("/"))
@@ -1248,6 +1240,7 @@ def hf_hub_download(
         library_name=library_name,
         library_version=library_version,
         user_agent=user_agent,
+        headers=headers,
     )
 
     url_to_download = url
@@ -1448,7 +1441,7 @@ def hf_hub_download(
         blob_path = "\\\\?\\" + os.path.abspath(blob_path)
 
     Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
-    with FileLock(lock_path):
+    with WeakFileLock(lock_path):
         # If the download just completed while the lock was activated.
         if os.path.exists(pointer_path) and not force_download:
             # Even if returning early like here, the lock will be released.
@@ -1630,6 +1623,7 @@ def get_hf_file_metadata(
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
     user_agent: Union[Dict, str, None] = None,
+    headers: Optional[Dict[str, str]] = None,
 ) -> HfFileMetadata:
     """Fetch metadata of a file versioned on the Hub for a given url.
 
@@ -1653,13 +1647,19 @@ def get_hf_file_metadata(
             The version of the library.
         user_agent (`dict`, `str`, *optional*):
             The user-agent info in the form of a dictionary or a string.
+        headers (`dict`, *optional*):
+            Additional headers to be sent with the request.
 
     Returns:
         A [`HfFileMetadata`] object containing metadata such as location, etag, size and
         commit_hash.
     """
     headers = build_hf_headers(
-        token=token, library_name=library_name, library_version=library_version, user_agent=user_agent
+        token=token,
+        library_name=library_name,
+        library_version=library_version,
+        user_agent=user_agent,
+        headers=headers,
     )
     headers["Accept-Encoding"] = "identity"  # prevent any compression => we want to know the real size of the file
 

huggingface_hub/hf_api.py

@@ -82,6 +82,8 @@ from .community import (
     deserialize_event,
 )
 from .constants import (
+    _HF_DEFAULT_ENDPOINT,
+    _HF_DEFAULT_STAGING_ENDPOINT,
     DEFAULT_ETAG_TIMEOUT,
     DEFAULT_REQUEST_TIMEOUT,
     DEFAULT_REVISION,
@@ -182,6 +184,12 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
             If `repo_type` is unknown.
     """
     input_hf_id = hf_id
+
+    # check if a proxy has been set => if yes, update the returned URL to use the proxy
+    if ENDPOINT not in (_HF_DEFAULT_ENDPOINT, _HF_DEFAULT_STAGING_ENDPOINT):
+        hf_id = hf_id.replace(_HF_DEFAULT_ENDPOINT, ENDPOINT)
+        hf_id = hf_id.replace(_HF_DEFAULT_STAGING_ENDPOINT, ENDPOINT)
+
     hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
     is_hf_url = hub_url in hf_id and "@" not in hf_id
 
@@ -1234,10 +1242,11 @@ class HfApi:
     def __init__(
         self,
         endpoint: Optional[str] = None,
-        token: Optional[str] = None,
+        token: Union[str, bool, None] = None,
         library_name: Optional[str] = None,
         library_version: Optional[str] = None,
         user_agent: Union[Dict, str, None] = None,
+        headers: Optional[Dict[str, str]] = None,
     ) -> None:
         """Create a HF client to interact with the Hub via HTTP.
 
@@ -1247,9 +1256,9 @@ class HfApi:
         directly at the root of `huggingface_hub`.
 
         Args:
-            token (`str`, *optional*):
-                Hugging Face token. Will default to the locally saved token if
-                not provided.
+            token (`str` or `bool`, *optional*):
+                Hugging Face token. Will default to the locally saved token if not provided.
+                Pass `token=False` if you don't want to send your token to the server.
             library_name (`str`, *optional*):
                 The name of the library that is making the HTTP request. Will be added to
                 the user-agent header. Example: `"transformers"`.
@@ -1259,12 +1268,16 @@ class HfApi:
             user_agent (`str`, `dict`, *optional*):
                 The user agent info in the form of a dictionary or a single string. It will
                 be completed with information about the installed packages.
+            headers (`dict`, *optional*):
+                Additional headers to be sent with each request. Example: `{"X-My-Header": "value"}`.
+                Headers passed here are taking precedence over the default headers.
         """
         self.endpoint = endpoint if endpoint is not None else ENDPOINT
         self.token = token
         self.library_name = library_name
         self.library_version = library_version
         self.user_agent = user_agent
+        self.headers = headers
         self._thread_pool: Optional[ThreadPoolExecutor] = None
 
     def run_as_future(self, fn: Callable[..., R], *args, **kwargs) -> Future[R]:
@@ -2514,7 +2527,7 @@ class HfApi:
         *,
         repo_type: Optional[str] = None,
         revision: Optional[str] = None,
-        token: Optional[str] = None,
+        token: Union[str, bool, None] = None,
     ) -> bool:
         """
         Checks if a file exists in a repository on the Hugging Face Hub.
@@ -3227,7 +3240,7 @@ class HfApi:
 
         # Prepare request
         url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/super-squash/{branch}"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         commit_message = commit_message or f"Super-squash branch '{branch}' using huggingface_hub"
 
         # Super-squash
@@ -3333,7 +3346,7 @@ class HfApi:
             # Testing purposes only.
             # See https://github.com/huggingface/huggingface_hub/pull/733/files#r820604472
             json["lfsmultipartthresh"] = self._lfsmultipartthresh  # type: ignore
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
 
         while True:
             r = get_session().post(path, headers=headers, json=json)
@@ -3408,7 +3421,7 @@ class HfApi:
         if repo_type is not None:
             json["type"] = repo_type
 
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         r = get_session().delete(path, headers=headers, json=json)
         try:
             hf_raise_for_status(r)
@@ -3473,7 +3486,7 @@ class HfApi:
 
         r = get_session().put(
             url=f"{self.endpoint}/api/{repo_type}s/{namespace}/{name}/settings",
-            headers=self._build_hf_headers(token=token, is_write_action=True),
+            headers=self._build_hf_headers(token=token),
             json={"private": private},
         )
         hf_raise_for_status(r)
@@ -3530,7 +3543,7 @@ class HfApi:
         json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type}
 
         path = f"{self.endpoint}/api/repos/move"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         r = get_session().post(path, headers=headers, json=json)
         try:
             hf_raise_for_status(r)
@@ -3556,8 +3569,7 @@ class HfApi:
         num_threads: int = 5,
         parent_commit: Optional[str] = None,
         run_as_future: Literal[False] = ...,
-    ) -> CommitInfo:
-        ...
+    ) -> CommitInfo: ...
 
     @overload
     def create_commit(
@@ -3574,8 +3586,7 @@ class HfApi:
         num_threads: int = 5,
         parent_commit: Optional[str] = None,
         run_as_future: Literal[True] = ...,
-    ) -> Future[CommitInfo]:
-        ...
+    ) -> Future[CommitInfo]: ...
 
     @validate_hf_hub_args
     @future_compatible
@@ -3711,6 +3722,8 @@ class HfApi:
         revision = quote(unquoted_revision, safe="")
         create_pr = create_pr if create_pr is not None else False
 
+        headers = self._build_hf_headers(token=token)
+
         operations = list(operations)
         additions = [op for op in operations if isinstance(op, CommitOperationAdd)]
         copies = [op for op in operations if isinstance(op, CommitOperationCopy)]
@@ -3738,7 +3751,7 @@ class HfApi:
                     response = get_session().post(
                         f"{ENDPOINT}/api/validate-yaml",
                         json={"content": file.read().decode(), "repoType": repo_type},
-                        headers=self._build_hf_headers(token=token),
+                        headers=headers,
                     )
                     # Handle warnings (example: empty metadata)
                     response_content = response.json()
@@ -3773,7 +3786,7 @@ class HfApi:
             copies=copies,
             repo_type=repo_type,
             repo_id=repo_id,
-            token=token or self.token,
+            headers=headers,
             revision=revision,
             endpoint=self.endpoint,
         )
@@ -3794,7 +3807,7 @@ class HfApi:
         headers = {
             # See https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073
             "Content-Type": "application/x-ndjson",
-            **self._build_hf_headers(token=token, is_write_action=True),
+            **headers,
         }
         data = b"".join(_payload_as_ndjson())
         params = {"create_pr": "1"} if create_pr else None
@@ -4213,6 +4226,7 @@ class HfApi:
             raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
         revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
         create_pr = create_pr if create_pr is not None else False
+        headers = self._build_hf_headers(token=token)
 
         # Check if a `gitignore` file is being committed to the Hub.
         additions = list(additions)
@@ -4232,7 +4246,7 @@ class HfApi:
                 additions=new_additions,
                 repo_type=repo_type,
                 repo_id=repo_id,
-                token=token or self.token,
+                headers=headers,
                 revision=revision,
                 endpoint=self.endpoint,
                 create_pr=create_pr or False,
@@ -4263,7 +4277,7 @@ class HfApi:
             additions=new_lfs_additions_to_upload,
             repo_type=repo_type,
             repo_id=repo_id,
-            token=token or self.token,
+            headers=headers,
             endpoint=self.endpoint,
             num_threads=num_threads,
             # If `create_pr`, we don't want to check user permission on the revision as users with read permission
@@ -4291,8 +4305,7 @@ class HfApi:
         create_pr: Optional[bool] = None,
         parent_commit: Optional[str] = None,
         run_as_future: Literal[False] = ...,
-    ) -> CommitInfo:
-        ...
+    ) -> CommitInfo: ...
 
     @overload
     def upload_file(
@@ -4309,8 +4322,7 @@ class HfApi:
         create_pr: Optional[bool] = None,
         parent_commit: Optional[str] = None,
         run_as_future: Literal[True] = ...,
-    ) -> Future[CommitInfo]:
-        ...
+    ) -> Future[CommitInfo]: ...
 
     @validate_hf_hub_args
     @future_compatible
@@ -4498,8 +4510,7 @@ class HfApi:
         multi_commits: Literal[False] = ...,
         multi_commits_verbose: bool = False,
         run_as_future: Literal[False] = ...,
-    ) -> CommitInfo:
-        ...
+    ) -> CommitInfo: ...
 
     @overload
     def upload_folder(  # type: ignore
@@ -4544,8 +4555,7 @@ class HfApi:
         multi_commits: Literal[False] = ...,
         multi_commits_verbose: bool = False,
         run_as_future: Literal[True] = ...,
-    ) -> Future[CommitInfo]:
-        ...
+    ) -> Future[CommitInfo]: ...
 
     @overload
     def upload_folder(
@@ -5188,6 +5198,7 @@ class HfApi:
             etag_timeout=etag_timeout,
             resume_download=resume_download,
             token=token,
+            headers=self.headers,
             local_files_only=local_files_only,
             legacy_cache_layout=legacy_cache_layout,
         )
@@ -5609,7 +5620,7 @@ class HfApi:
 
         # Prepare request
         branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         payload = {}
         if revision is not None:
             payload["startingPoint"] = revision
@@ -5665,7 +5676,7 @@ class HfApi:
 
         # Prepare request
         branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
 
         # Delete branch
         response = get_session().delete(url=branch_url, headers=headers)
@@ -5729,7 +5740,7 @@ class HfApi:
 
         # Prepare request
         tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{revision}"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         payload = {"tag": tag}
         if tag_message is not None:
             payload["message"] = tag_message
@@ -5782,7 +5793,7 @@ class HfApi:
 
         # Prepare request
         tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{tag}"
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
 
         # Un-tag
         response = get_session().delete(url=tag_url, headers=headers)
@@ -6077,7 +6088,7 @@ class HfApi:
             )
         )
 
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         resp = get_session().post(
             f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions",
             json={
@@ -6175,7 +6186,7 @@ class HfApi:
 
         path = f"{self.endpoint}/api/{repo_id}/discussions/{discussion_num}/{resource}"
 
-        headers = self._build_hf_headers(token=token, is_write_action=True)
+        headers = self._build_hf_headers(token=token)
         resp = requests.post(path, headers=headers, json=body)
         hf_raise_for_status(resp)
         return resp
@@ -6993,7 +7004,7 @@ class HfApi:
 
         r = get_session().post(
             f"{self.endpoint}/api/spaces/{from_id}/duplicate",
-            headers=self._build_hf_headers(token=token, is_write_action=True),
+            headers=self._build_hf_headers(token=token),
             json=payload,
         )
 
@@ -8408,6 +8419,7 @@ class HfApi:
             library_name=library_name or self.library_name,
             library_version=library_version or self.library_version,
             user_agent=user_agent or self.user_agent,
+            headers=self.headers,
         )
 
     def _prepare_upload_folder_deletions(
@@ -8435,7 +8447,7 @@ class HfApi:
         filenames = self.list_repo_files(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token)
 
         # Compute relative path in repo
-        if path_in_repo:
+        if path_in_repo and path_in_repo not in (".", "./"):
             path_in_repo = path_in_repo.strip("/") + "/"  # harmonize
             relpath_to_abspath = {
                 file[len(path_in_repo) :]: file for file in filenames if file.startswith(path_in_repo)