huggingface-hub 0.31.0rc0__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/utils/_validators.py

@@ -19,7 +19,7 @@ import re
 import warnings
 from functools import wraps
 from itertools import chain
-from typing import Any, Dict
+from typing import Any
 
 from huggingface_hub.errors import HFValidationError
 
@@ -48,9 +48,7 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
     Validators:
         - [`~utils.validate_repo_id`]: `repo_id` must be `"repo_name"`
           or `"namespace/repo_name"`. Namespace is a username or an organization.
-        - [`~utils.smoothly_deprecate_use_auth_token`]: Use `token` instead of
-          `use_auth_token` (only if `use_auth_token` is not expected by the decorated
-          function - in practice, always the case in `huggingface_hub`).
+        - [`~utils.smoothly_deprecate_legacy_arguments`]: Ignore `proxies` when downloading files (should be set globally).
 
     Example:
     ```py
@@ -68,20 +66,6 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
 
     >>> my_cool_method(repo_id="other..repo..id")
     huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'.
-
-    >>> @validate_hf_hub_args
-    ... def my_cool_auth_method(token: str):
-    ...     print(token)
-
-    >>> my_cool_auth_method(token="a token")
-    "a token"
-
-    >>> my_cool_auth_method(use_auth_token="a use_auth_token")
-    "a use_auth_token"
-
-    >>> my_cool_auth_method(token="a token", use_auth_token="a use_auth_token")
-    UserWarning: Both `token` and `use_auth_token` are passed (...)
-    "a token"
     ```
 
     Raises:
@@ -91,13 +75,8 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
     # TODO: add an argument to opt-out validation for specific argument?
     signature = inspect.signature(fn)
 
-    # Should the validator switch `use_auth_token` values to `token`? In practice, always
-    # True in `huggingface_hub`. Might not be the case in a downstream library.
-    check_use_auth_token = "use_auth_token" not in signature.parameters and "token" in signature.parameters
-
     @wraps(fn)
     def _inner_fn(*args, **kwargs):
-        has_token = False
         for arg_name, arg_value in chain(
             zip(signature.parameters, args),  # Args values
             kwargs.items(),  # Kwargs values
@@ -105,11 +84,7 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
             if arg_name in ["repo_id", "from_id", "to_id"]:
                 validate_repo_id(arg_value)
 
-            elif arg_name == "token" and arg_value is not None:
-                has_token = True
-
-        if check_use_auth_token:
-            kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs)
+        kwargs = smoothly_deprecate_legacy_arguments(fn_name=fn.__name__, kwargs=kwargs)
 
         return fn(*args, **kwargs)
 
@@ -158,8 +133,8 @@ def validate_repo_id(repo_id: str) -> None:
 
     if not REPO_ID_REGEX.match(repo_id):
         raise HFValidationError(
-            "Repo id must use alphanumeric chars or '-', '_', '.', '--' and '..' are"
-            " forbidden, '-' and '.' cannot start or end the name, max length is 96:"
+            "Repo id must use alphanumeric chars, '-', '_' or '.'."
+            " The name cannot start or end with '-' or '.' and the maximum length is 96:"
             f" '{repo_id}'."
         )
 
@@ -170,57 +145,63 @@ def validate_repo_id(repo_id: str) -> None:
         raise HFValidationError(f"Repo_id cannot end by '.git': '{repo_id}'.")
 
 
-def smoothly_deprecate_use_auth_token(fn_name: str, has_token: bool, kwargs: Dict[str, Any]) -> Dict[str, Any]:
-    """Smoothly deprecate `use_auth_token` in the `huggingface_hub` codebase.
-
-    The long-term goal is to remove any mention of `use_auth_token` in the codebase in
-    favor of a unique and less verbose `token` argument. This will be done a few steps:
+def smoothly_deprecate_legacy_arguments(fn_name: str, kwargs: dict[str, Any]) -> dict[str, Any]:
+    """Smoothly deprecate legacy arguments in the `huggingface_hub` codebase.
 
-    0. Step 0: methods that require a read-access to the Hub use the `use_auth_token`
-       argument (`str`, `bool` or `None`). Methods requiring write-access have a `token`
-       argument (`str`, `None`). This implicit rule exists to be able to not send the
-       token when not necessary (`use_auth_token=False`) even if logged in.
+    This function ignores some deprecated arguments from the kwargs and warns the user they are ignored.
+    The goal is to avoid breaking existing code while guiding the user to the new way of doing things.
 
-    1. Step 1: we want to harmonize everything and use `token` everywhere (supporting
-       `token=False` for read-only methods). In order not to break existing code, if
-       `use_auth_token` is passed to a function, the `use_auth_token` value is passed
-       as `token` instead, without any warning.
-       a. Corner case: if both `use_auth_token` and `token` values are passed, a warning
-          is thrown and the `use_auth_token` value is ignored.
+    List of deprecated arguments:
+        - `proxies`:
+            To set up proxies, user must either use the HTTP_PROXY environment variable or configure the `httpx.Client`
+            manually using the [`set_client_factory`] function.
 
-    2. Step 2: Once it is release, we should push downstream libraries to switch from
-       `use_auth_token` to `token` as much as possible, but without throwing a warning
-       (e.g. manually create issues on the corresponding repos).
+            In huggingface_hub 0.x, `proxies` was a dictionary directly passed to `requests.request`.
+            In huggingface_hub 1.x, we migrated to `httpx` which does not support `proxies` the same way.
+            In particular, it is not possible to configure proxies on a per-request basis. The solution is to configure
+            it globally using the [`set_client_factory`] function or using the HTTP_PROXY environment variable.
 
-    3. Step 3: After a transitional period (6 months e.g. until April 2023?), we update
-       `huggingface_hub` to throw a warning on `use_auth_token`. Hopefully, very few
-       users will be impacted as it would have already been fixed.
-       In addition, unit tests in `huggingface_hub` must be adapted to expect warnings
-       to be thrown (but still use `use_auth_token` as before).
+            For more details, see:
+            - https://www.python-httpx.org/advanced/proxies/
+            - https://www.python-httpx.org/compatibility/#proxy-keys.
 
-    4. Step 4: After a normal deprecation cycle (3 releases ?), remove this validator.
-       `use_auth_token` will definitely not be supported.
-       In addition, we update unit tests in `huggingface_hub` to use `token` everywhere.
-
-    This has been discussed in:
-    - https://github.com/huggingface/huggingface_hub/issues/1094.
-    - https://github.com/huggingface/huggingface_hub/pull/928
-    - (related) https://github.com/huggingface/huggingface_hub/pull/1064
+        - `resume_download`: deprecated without replacement. `huggingface_hub` always resumes downloads whenever possible.
+        - `force_filename`: deprecated without replacement. Filename is always the same as on the Hub.
+        - `local_dir_use_symlinks`: deprecated without replacement. Downloading to a local directory does not use symlinks anymore.
     """
     new_kwargs = kwargs.copy()  # do not mutate input !
 
-    use_auth_token = new_kwargs.pop("use_auth_token", None)  # remove from kwargs
-    if use_auth_token is not None:
-        if has_token:
-            warnings.warn(
-                "Both `token` and `use_auth_token` are passed to"
-                f" `{fn_name}` with non-None values. `token` is now the"
-                " preferred argument to pass a User Access Token."
-                " `use_auth_token` value will be ignored."
-            )
-        else:
-            # `token` argument is not passed and a non-None value is passed in
-            # `use_auth_token` => use `use_auth_token` value as `token` kwarg.
-            new_kwargs["token"] = use_auth_token
+    # proxies
+    proxies = new_kwargs.pop("proxies", None)  # remove from kwargs
+    if proxies is not None:
+        warnings.warn(
+            f"The `proxies` argument is ignored in `{fn_name}`. To set up proxies, use the HTTP_PROXY / HTTPS_PROXY"
+            " environment variables or configure the `httpx.Client` manually using `huggingface_hub.set_client_factory`."
+            " See https://www.python-httpx.org/advanced/proxies/ for more details."
+        )
+
+    # resume_download
+    resume_download = new_kwargs.pop("resume_download", None)  # remove from kwargs
+    if resume_download is not None:
+        warnings.warn(
+            f"The `resume_download` argument is deprecated and ignored in `{fn_name}`. Downloads always resume"
+            " whenever possible."
+        )
+
+    # force_filename
+    force_filename = new_kwargs.pop("force_filename", None)  # remove from kwargs
+    if force_filename is not None:
+        warnings.warn(
+            f"The `force_filename` argument is deprecated and ignored in `{fn_name}`. Filename is always the same "
+            "as on the Hub."
+        )
+
+    # local_dir_use_symlinks
+    local_dir_use_symlinks = new_kwargs.pop("local_dir_use_symlinks", None)  # remove from kwargs
+    if local_dir_use_symlinks is not None:
+        warnings.warn(
+            f"The `local_dir_use_symlinks` argument is deprecated and ignored in `{fn_name}`. Downloading to a local"
+            " directory does not use symlinks anymore."
+        )
 
     return new_kwargs

huggingface_hub/utils/_verification.py

@@ -0,0 +1,167 @@
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal, Optional, TypedDict, Union
+
+from .. import constants
+from ..file_download import repo_folder_name
+from .sha import git_hash, sha_fileobj
+
+
+if TYPE_CHECKING:
+    from ..hf_api import RepoFile, RepoFolder
+
+# using fullmatch for clarity and strictness
+_REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$")
+
+
+# Typed structure describing a checksum mismatch
+class Mismatch(TypedDict):
+    path: str
+    expected: str
+    actual: str
+    algorithm: str
+
+
+HashAlgo = Literal["sha256", "git-sha1"]
+
+
+@dataclass(frozen=True)
+class FolderVerification:
+    revision: str
+    checked_count: int
+    mismatches: list[Mismatch]
+    missing_paths: list[str]
+    extra_paths: list[str]
+    verified_path: Path
+
+
+def collect_local_files(root: Path) -> dict[str, Path]:
+    """
+    Return a mapping of repo-relative path -> absolute path for all files under `root`.
+    """
+    return {p.relative_to(root).as_posix(): p for p in root.rglob("*") if p.is_file()}
+
+
+def _resolve_commit_hash_from_cache(storage_folder: Path, revision: Optional[str]) -> str:
+    """
+    Resolve a commit hash from a cache repo folder and an optional revision.
+    """
+    if revision and _REGEX_COMMIT_HASH.fullmatch(revision):
+        return revision
+
+    refs_dir = storage_folder / "refs"
+    snapshots_dir = storage_folder / "snapshots"
+
+    if revision:
+        ref_path = refs_dir / revision
+        if ref_path.is_file():
+            return ref_path.read_text(encoding="utf-8").strip()
+        raise ValueError(f"Revision '{revision}' could not be resolved in cache (expected file '{ref_path}').")
+
+    # No revision provided: try common defaults
+    main_ref = refs_dir / "main"
+    if main_ref.is_file():
+        return main_ref.read_text(encoding="utf-8").strip()
+
+    if not snapshots_dir.is_dir():
+        raise ValueError(f"Cache repo is missing snapshots directory: {snapshots_dir}. Provide --revision explicitly.")
+
+    candidates = [p.name for p in snapshots_dir.iterdir() if p.is_dir() and _REGEX_COMMIT_HASH.fullmatch(p.name)]
+    if len(candidates) == 1:
+        return candidates[0]
+
+    raise ValueError(
+        "Ambiguous cached revision: multiple snapshots found and no refs to disambiguate. Please pass --revision."
+    )
+
+
+def compute_file_hash(path: Path, algorithm: HashAlgo) -> str:
+    """
+    Compute the checksum of a local file using the requested algorithm.
+    """
+
+    with path.open("rb") as stream:
+        if algorithm == "sha256":
+            return sha_fileobj(stream).hex()
+        if algorithm == "git-sha1":
+            return git_hash(stream.read())
+        raise ValueError(f"Unsupported hash algorithm: {algorithm}")
+
+
+def verify_maps(
+    *,
+    remote_by_path: dict[str, Union["RepoFile", "RepoFolder"]],
+    local_by_path: dict[str, Path],
+    revision: str,
+    verified_path: Path,
+) -> FolderVerification:
+    """Compare remote entries and local files and return a verification result."""
+    remote_paths = set(remote_by_path)
+    local_paths = set(local_by_path)
+
+    missing = sorted(remote_paths - local_paths)
+    extra = sorted(local_paths - remote_paths)
+    both = sorted(remote_paths & local_paths)
+
+    mismatches: list[Mismatch] = []
+
+    for rel_path in both:
+        remote_entry = remote_by_path[rel_path]
+        local_path = local_by_path[rel_path]
+
+        lfs = getattr(remote_entry, "lfs", None)
+        lfs_sha = getattr(lfs, "sha256", None) if lfs is not None else None
+        if lfs_sha is None and isinstance(lfs, dict):
+            lfs_sha = lfs.get("sha256")
+        if lfs_sha:
+            algorithm: HashAlgo = "sha256"
+            expected = str(lfs_sha).lower()
+        else:
+            blob_id = remote_entry.blob_id  # type: ignore
+            algorithm = "git-sha1"
+            expected = str(blob_id).lower()
+
+        actual = compute_file_hash(local_path, algorithm)
+
+        if actual != expected:
+            mismatches.append(Mismatch(path=rel_path, expected=expected, actual=actual, algorithm=algorithm))
+
+    return FolderVerification(
+        revision=revision,
+        checked_count=len(both),
+        mismatches=mismatches,
+        missing_paths=missing,
+        extra_paths=extra,
+        verified_path=verified_path,
+    )
+
+
+def resolve_local_root(
+    *,
+    repo_id: str,
+    repo_type: str,
+    revision: Optional[str],
+    cache_dir: Optional[Path],
+    local_dir: Optional[Path],
+) -> tuple[Path, str]:
+    """
+    Resolve the root directory to scan locally and the remote revision to verify.
+    """
+    if local_dir is not None:
+        root = Path(local_dir).expanduser().resolve()
+        if not root.is_dir():
+            raise ValueError(f"Local directory does not exist or is not a directory: {root}")
+        return root, (revision or constants.DEFAULT_REVISION)
+
+    cache_root = Path(cache_dir or constants.HF_HUB_CACHE).expanduser().resolve()
+    storage_folder = cache_root / repo_folder_name(repo_id=repo_id, repo_type=repo_type)
+    if not storage_folder.exists():
+        raise ValueError(
+            f"Repo is not present in cache: {storage_folder}. Use 'hf download' first or pass --local-dir."
+        )
+    commit = _resolve_commit_hash_from_cache(storage_folder, revision)
+    snapshot_dir = storage_folder / "snapshots" / commit
+    if not snapshot_dir.is_dir():
+        raise ValueError(f"Snapshot directory does not exist for revision '{commit}': {snapshot_dir}.")
+    return snapshot_dir, commit

huggingface_hub/utils/_xet.py

@@ -1,13 +1,19 @@
+import time
 from dataclasses import dataclass
 from enum import Enum
-from typing import Dict, Optional
+from typing import Optional
 
-import requests
+import httpx
 
 from .. import constants
 from . import get_session, hf_raise_for_status, validate_hf_hub_args
 
 
+XET_CONNECTION_INFO_SAFETY_PERIOD = 60  # seconds
+XET_CONNECTION_INFO_CACHE_SIZE = 1_000
+XET_CONNECTION_INFO_CACHE: dict[str, "XetConnectionInfo"] = {}
+
+
 class XetTokenType(str, Enum):
     READ = "read"
     WRITE = "write"
@@ -26,7 +32,9 @@ class XetConnectionInfo:
     endpoint: str
 
 
-def parse_xet_file_data_from_response(response: requests.Response) -> Optional[XetFileData]:
+def parse_xet_file_data_from_response(
+    response: httpx.Response, endpoint: Optional[str] = None
+) -> Optional[XetFileData]:
     """
     Parse XET file metadata from an HTTP response.
 
@@ -34,7 +42,7 @@ def parse_xet_file_data_from_response(response: requests.Response) -> Optional[X
     of a given response object. If the required metadata is not found, it returns `None`.
 
     Args:
-        response (`requests.Response`):
+        response (`httpx.Response`):
             The HTTP response object containing headers dict and links dict to extract the XET metadata from.
     Returns:
         `Optional[XetFileData]`:
@@ -52,18 +60,20 @@ def parse_xet_file_data_from_response(response: requests.Response) -> Optional[X
             refresh_route = response.headers[constants.HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE]
     except KeyError:
         return None
-
+    endpoint = endpoint if endpoint is not None else constants.ENDPOINT
+    if refresh_route.startswith(constants.HUGGINGFACE_CO_URL_HOME):
+        refresh_route = refresh_route.replace(constants.HUGGINGFACE_CO_URL_HOME.rstrip("/"), endpoint.rstrip("/"))
     return XetFileData(
         file_hash=file_hash,
         refresh_route=refresh_route,
     )
 
 
-def parse_xet_connection_info_from_headers(headers: Dict[str, str]) -> Optional[XetConnectionInfo]:
+def parse_xet_connection_info_from_headers(headers: dict[str, str]) -> Optional[XetConnectionInfo]:
     """
     Parse XET connection info from the HTTP headers or return None if not found.
     Args:
-        headers (`Dict`):
+        headers (`dict`):
            HTTP headers to extract the XET metadata from.
     Returns:
         `XetConnectionInfo` or `None`:
@@ -88,7 +98,7 @@ def parse_xet_connection_info_from_headers(headers: Dict[str, str]) -> Optional[
 def refresh_xet_connection_info(
     *,
     file_data: XetFileData,
-    headers: Dict[str, str],
+    headers: dict[str, str],
 ) -> XetConnectionInfo:
     """
     Utilizes the information in the parsed metadata to request the Hub xet connection information.
@@ -96,7 +106,7 @@ def refresh_xet_connection_info(
     Args:
         file_data: (`XetFileData`):
             The file data needed to refresh the xet connection information.
-        headers (`Dict[str, str]`):
+        headers (`dict[str, str]`):
             Headers to use for the request, including authorization headers and user agent.
     Returns:
         `XetConnectionInfo`:
@@ -119,9 +129,9 @@ def fetch_xet_connection_info_from_repo_info(
     repo_id: str,
     repo_type: str,
     revision: Optional[str] = None,
-    headers: Dict[str, str],
+    headers: dict[str, str],
     endpoint: Optional[str] = None,
-    params: Optional[Dict[str, str]] = None,
+    params: Optional[dict[str, str]] = None,
 ) -> XetConnectionInfo:
     """
     Uses the repo info to request a xet access token from Hub.
@@ -134,11 +144,11 @@ def fetch_xet_connection_info_from_repo_info(
             Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
         revision (`str`, `optional`):
             The revision of the repo to get the token for.
-        headers (`Dict[str, str]`):
+        headers (`dict[str, str]`):
             Headers to use for the request, including authorization headers and user agent.
         endpoint (`str`, `optional`):
             The endpoint to use for the request. Defaults to the Hub endpoint.
-        params (`Dict[str, str]`, `optional`):
+        params (`dict[str, str]`, `optional`):
             Additional parameters to pass with the request.
     Returns:
         `XetConnectionInfo`:
@@ -157,18 +167,21 @@ def fetch_xet_connection_info_from_repo_info(
 @validate_hf_hub_args
 def _fetch_xet_connection_info_with_url(
     url: str,
-    headers: Dict[str, str],
-    params: Optional[Dict[str, str]] = None,
+    headers: dict[str, str],
+    params: Optional[dict[str, str]] = None,
 ) -> XetConnectionInfo:
     """
     Requests the xet connection info from the supplied URL. This includes the
     access token, expiration time, and endpoint to use for the xet storage service.
+
+    Result is cached to avoid redundant requests.
+
     Args:
         url: (`str`):
             The access token endpoint URL.
-        headers (`Dict[str, str]`):
+        headers (`dict[str, str]`):
             Headers to use for the request, including authorization headers and user agent.
-        params (`Dict[str, str]`, `optional`):
+        params (`dict[str, str]`, `optional`):
             Additional parameters to pass with the request.
     Returns:
         `XetConnectionInfo`:
@@ -179,10 +192,44 @@ def _fetch_xet_connection_info_with_url(
         [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
             If the Hub API response is improperly formatted.
     """
+    # Check cache first
+    cache_key = _cache_key(url, headers, params)
+    cached_info = XET_CONNECTION_INFO_CACHE.get(cache_key)
+    if cached_info is not None:
+        if not _is_expired(cached_info):
+            return cached_info
+
+    # Fetch from server
     resp = get_session().get(headers=headers, url=url, params=params)
     hf_raise_for_status(resp)
 
     metadata = parse_xet_connection_info_from_headers(resp.headers)  # type: ignore
     if metadata is None:
         raise ValueError("Xet headers have not been correctly set by the server.")
+
+    # Delete expired cache entries
+    for k, v in list(XET_CONNECTION_INFO_CACHE.items()):
+        if _is_expired(v):
+            XET_CONNECTION_INFO_CACHE.pop(k, None)
+
+    # Enforce cache size limit
+    if len(XET_CONNECTION_INFO_CACHE) >= XET_CONNECTION_INFO_CACHE_SIZE:
+        XET_CONNECTION_INFO_CACHE.pop(next(iter(XET_CONNECTION_INFO_CACHE)))
+
+    # Update cache
+    XET_CONNECTION_INFO_CACHE[cache_key] = metadata
+
     return metadata
+
+
+def _cache_key(url: str, headers: dict[str, str], params: Optional[dict[str, str]]) -> str:
+    """Return a unique cache key for the given request parameters."""
+    lower_headers = {k.lower(): v for k, v in headers.items()}  # casing is not guaranteed here
+    auth_header = lower_headers.get("authorization", "")
+    params_str = "&".join(f"{k}={v}" for k, v in sorted((params or {}).items(), key=lambda x: x[0]))
+    return f"{url}|{auth_header}|{params_str}"
+
+
+def _is_expired(connection_info: XetConnectionInfo) -> bool:
+    """Check if the given XET connection info is expired."""
+    return connection_info.expiration_unix_epoch <= int(time.time()) + XET_CONNECTION_INFO_SAFETY_PERIOD