huggingface-hub 0.22.0rc0__tar.gz → 0.22.2__tar.gz

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: huggingface_hub
-Version: 0.22.0rc0
+Version: 0.22.2
 Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
 Home-page: https://github.com/huggingface/huggingface_hub
 Author: Hugging Face, Inc.

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.22.0.rc0"
+__version__ = "0.22.2"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/_commit_api.py

@@ -399,7 +399,7 @@ def _upload_lfs_files(
     def _wrapped_lfs_upload(batch_action) -> None:
         try:
             operation = oid2addop[batch_action["oid"]]
-            lfs_upload(operation=operation, lfs_batch_action=batch_action, headers=headers)
+            lfs_upload(operation=operation, lfs_batch_action=batch_action, headers=headers, endpoint=endpoint)
         except Exception as exc:
             raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc
 

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/file_download.py

@@ -404,6 +404,7 @@ def http_get(
     expected_size: Optional[int] = None,
     displayed_filename: Optional[str] = None,
     _nb_retries: int = 5,
+    _tqdm_bar: Optional[tqdm] = None,
 ) -> None:
     """
     Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub.
@@ -483,84 +484,90 @@ def http_get(
     )
 
     # Stream file to buffer
-    with tqdm(
-        unit="B",
-        unit_scale=True,
-        total=total,
-        initial=resume_size,
-        desc=displayed_filename,
-        disable=True if (logger.getEffectiveLevel() == logging.NOTSET) else None,
-        # ^ set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
-        # see https://github.com/huggingface/huggingface_hub/pull/2000
-    ) as progress:
-        if hf_transfer and total is not None and total > 5 * DOWNLOAD_CHUNK_SIZE:
-            supports_callback = "callback" in inspect.signature(hf_transfer.download).parameters
-            if not supports_callback:
-                warnings.warn(
-                    "You are using an outdated version of `hf_transfer`. "
-                    "Consider upgrading to latest version to enable progress bars "
-                    "using `pip install -U hf_transfer`."
-                )
-            try:
-                hf_transfer.download(
-                    url=url,
-                    filename=temp_file.name,
-                    max_files=HF_TRANSFER_CONCURRENCY,
-                    chunk_size=DOWNLOAD_CHUNK_SIZE,
-                    headers=headers,
-                    parallel_failures=3,
-                    max_retries=5,
-                    **({"callback": progress.update} if supports_callback else {}),
-                )
-            except Exception as e:
-                raise RuntimeError(
-                    "An error occurred while downloading using `hf_transfer`. Consider"
-                    " disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling."
-                ) from e
-            if not supports_callback:
-                progress.update(total)
-            if expected_size is not None and expected_size != os.path.getsize(temp_file.name):
-                raise EnvironmentError(
-                    consistency_error_message.format(
-                        actual_size=os.path.getsize(temp_file.name),
-                    )
-                )
-            return
-        new_resume_size = resume_size
+    progress = _tqdm_bar
+    if progress is None:
+        progress = tqdm(
+            unit="B",
+            unit_scale=True,
+            total=total,
+            initial=resume_size,
+            desc=displayed_filename,
+            disable=True if (logger.getEffectiveLevel() == logging.NOTSET) else None,
+            # ^ set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+            # see https://github.com/huggingface/huggingface_hub/pull/2000
+        )
+
+    if hf_transfer and total is not None and total > 5 * DOWNLOAD_CHUNK_SIZE:
+        supports_callback = "callback" in inspect.signature(hf_transfer.download).parameters
+        if not supports_callback:
+            warnings.warn(
+                "You are using an outdated version of `hf_transfer`. "
+                "Consider upgrading to latest version to enable progress bars "
+                "using `pip install -U hf_transfer`."
+            )
         try:
-            for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
-                if chunk:  # filter out keep-alive new chunks
-                    progress.update(len(chunk))
-                    temp_file.write(chunk)
-                    new_resume_size += len(chunk)
-                    # Some data has been downloaded from the server so we reset the number of retries.
-                    _nb_retries = 5
-        except (requests.ConnectionError, requests.ReadTimeout) as e:
-            # If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely
-            # a transient error (network outage?). We log a warning message and try to resume the download a few times
-            # before giving up. Tre retry mechanism is basic but should be enough in most cases.
-            if _nb_retries <= 0:
-                logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e))
-                raise
-            logger.warning("Error while downloading from %s: %s\nTrying to resume download...", url, str(e))
-            time.sleep(1)
-            reset_sessions()  # In case of SSLError it's best to reset the shared requests.Session objects
-            return http_get(
+            hf_transfer.download(
                 url=url,
-                temp_file=temp_file,
-                proxies=proxies,
-                resume_size=new_resume_size,
-                headers=initial_headers,
-                expected_size=expected_size,
-                _nb_retries=_nb_retries - 1,
+                filename=temp_file.name,
+                max_files=HF_TRANSFER_CONCURRENCY,
+                chunk_size=DOWNLOAD_CHUNK_SIZE,
+                headers=headers,
+                parallel_failures=3,
+                max_retries=5,
+                **({"callback": progress.update} if supports_callback else {}),
             )
-
-        if expected_size is not None and expected_size != temp_file.tell():
+        except Exception as e:
+            raise RuntimeError(
+                "An error occurred while downloading using `hf_transfer`. Consider"
+                " disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling."
+            ) from e
+        if not supports_callback:
+            progress.update(total)
+        if expected_size is not None and expected_size != os.path.getsize(temp_file.name):
             raise EnvironmentError(
                 consistency_error_message.format(
-                    actual_size=temp_file.tell(),
+                    actual_size=os.path.getsize(temp_file.name),
                 )
             )
+        return
+    new_resume_size = resume_size
+    try:
+        for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
+            if chunk:  # filter out keep-alive new chunks
+                progress.update(len(chunk))
+                temp_file.write(chunk)
+                new_resume_size += len(chunk)
+                # Some data has been downloaded from the server so we reset the number of retries.
+                _nb_retries = 5
+    except (requests.ConnectionError, requests.ReadTimeout) as e:
+        # If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely
+        # a transient error (network outage?). We log a warning message and try to resume the download a few times
+        # before giving up. Tre retry mechanism is basic but should be enough in most cases.
+        if _nb_retries <= 0:
+            logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e))
+            raise
+        logger.warning("Error while downloading from %s: %s\nTrying to resume download...", url, str(e))
+        time.sleep(1)
+        reset_sessions()  # In case of SSLError it's best to reset the shared requests.Session objects
+        return http_get(
+            url=url,
+            temp_file=temp_file,
+            proxies=proxies,
+            resume_size=new_resume_size,
+            headers=initial_headers,
+            expected_size=expected_size,
+            _nb_retries=_nb_retries - 1,
+            _tqdm_bar=_tqdm_bar,
+        )
+
+    progress.close()
+
+    if expected_size is not None and expected_size != temp_file.tell():
+        raise EnvironmentError(
+            consistency_error_message.format(
+                actual_size=temp_file.tell(),
+            )
+        )
 
 
 @validate_hf_hub_args

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/hf_api.py

@@ -82,8 +82,6 @@ from .community import (
     deserialize_event,
 )
 from .constants import (
-    _HF_DEFAULT_ENDPOINT,
-    _HF_DEFAULT_STAGING_ENDPOINT,
     DEFAULT_ETAG_TIMEOUT,
     DEFAULT_REQUEST_TIMEOUT,
     DEFAULT_REVISION,
@@ -123,6 +121,7 @@ from .utils import (  # noqa: F401 # imported for backward compatibility
     build_hf_headers,
     experimental,
     filter_repo_objects,
+    fix_hf_endpoint_in_url,
     get_session,
     hf_raise_for_status,
     logging,
@@ -185,11 +184,6 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
     """
     input_hf_id = hf_id
 
-    # check if a proxy has been set => if yes, update the returned URL to use the proxy
-    if ENDPOINT not in (_HF_DEFAULT_ENDPOINT, _HF_DEFAULT_STAGING_ENDPOINT):
-        hf_id = hf_id.replace(_HF_DEFAULT_ENDPOINT, ENDPOINT)
-        hf_id = hf_id.replace(_HF_DEFAULT_STAGING_ENDPOINT, ENDPOINT)
-
     hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
     is_hf_url = hub_url in hf_id and "@" not in hf_id
 
@@ -436,6 +430,7 @@ class RepoUrl(str):
     """
 
     def __new__(cls, url: Any, endpoint: Optional[str] = None):
+        url = fix_hf_endpoint_in_url(url, endpoint=endpoint)
         return super(RepoUrl, cls).__new__(cls, url)
 
     def __init__(self, url: Any, endpoint: Optional[str] = None) -> None:

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/hf_file_system.py

@@ -6,15 +6,24 @@ from collections import deque
 from dataclasses import dataclass, field
 from datetime import datetime
 from itertools import chain
+from pathlib import Path
 from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
 from urllib.parse import quote, unquote
 
 import fsspec
+from fsspec.callbacks import _DEFAULT_CALLBACK, NoOpCallback, TqdmCallback
+from fsspec.utils import isfilelike
 from requests import Response
 
 from ._commit_api import CommitOperationCopy, CommitOperationDelete
-from .constants import DEFAULT_REVISION, ENDPOINT, REPO_TYPE_MODEL, REPO_TYPES_MAPPING, REPO_TYPES_URL_PREFIXES
-from .file_download import hf_hub_url
+from .constants import (
+    DEFAULT_REVISION,
+    ENDPOINT,
+    REPO_TYPE_MODEL,
+    REPO_TYPES_MAPPING,
+    REPO_TYPES_URL_PREFIXES,
+)
+from .file_download import hf_hub_url, http_get
 from .hf_api import HfApi, LastCommitInfo, RepoFile
 from .utils import (
     EntryNotFoundError,
@@ -591,6 +600,58 @@ class HfFileSystem(fsspec.AbstractFileSystem):
             url = url.replace("/resolve/", "/tree/", 1)
         return url
 
+    def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs) -> None:
+        """Copy single remote file to local."""
+        revision = kwargs.get("revision")
+        unhandled_kwargs = set(kwargs.keys()) - {"revision"}
+        if not isinstance(callback, (NoOpCallback, TqdmCallback)) or len(unhandled_kwargs) > 0:
+            # for now, let's not handle custom callbacks
+            # and let's not handle custom kwargs
+            return super().get_file(rpath, lpath, callback=callback, outfile=outfile, **kwargs)
+
+        # Taken from https://github.com/fsspec/filesystem_spec/blob/47b445ae4c284a82dd15e0287b1ffc410e8fc470/fsspec/spec.py#L883
+        if isfilelike(lpath):
+            outfile = lpath
+        elif self.isdir(rpath):
+            os.makedirs(lpath, exist_ok=True)
+            return None
+
+        if isinstance(lpath, (str, Path)):  # otherwise, let's assume it's a file-like object
+            os.makedirs(os.path.dirname(lpath), exist_ok=True)
+
+        # Open file if not already open
+        close_file = False
+        if outfile is None:
+            outfile = open(lpath, "wb")
+            close_file = True
+        initial_pos = outfile.tell()
+
+        # Custom implementation of `get_file` to use `http_get`.
+        resolve_remote_path = self.resolve_path(rpath, revision=revision)
+        expected_size = self.info(rpath, revision=revision)["size"]
+        callback.set_size(expected_size)
+        try:
+            http_get(
+                url=hf_hub_url(
+                    repo_id=resolve_remote_path.repo_id,
+                    revision=resolve_remote_path.revision,
+                    filename=resolve_remote_path.path_in_repo,
+                    repo_type=resolve_remote_path.repo_type,
+                    endpoint=self.endpoint,
+                ),
+                temp_file=outfile,
+                displayed_filename=rpath,
+                expected_size=expected_size,
+                resume_size=0,
+                headers=self._api._build_hf_headers(),
+                _tqdm_bar=callback.tqdm if isinstance(callback, TqdmCallback) else None,
+            )
+            outfile.seek(initial_pos)
+        finally:
+            # Close file only if we opened it ourselves
+            if close_file:
+                outfile.close()
+
     @property
     def transaction(self):
         """A context within which files are committed together upon exit
@@ -618,6 +679,7 @@ class HfFileSystemFile(fsspec.spec.AbstractBufferedFile):
                 raise FileNotFoundError(
                     f"{e}.\nMake sure the repository and revision exist before writing data."
                 ) from e
+            raise
         super().__init__(fs, self.resolved_path.unresolve(), **kwargs)
         self.fs: HfFileSystem
 
@@ -667,6 +729,18 @@ class HfFileSystemFile(fsspec.spec.AbstractBufferedFile):
                 path=self.resolved_path.unresolve(),
             )
 
+    def read(self, length=-1):
+        """Read remote file.
+
+        If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if
+        `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a
+        temporary file and read from there.
+        """
+        if self.mode == "rb" and (length is None or length == -1) and self.loc == 0:
+            with self.fs.open(self.path, "rb", block_size=0) as f:  # block_size=0 enables fast streaming
+                return f.read()
+        return super().read(length)
+
     def url(self) -> str:
         return self.fs.url(self.path)
 
@@ -695,7 +769,7 @@ class HfFileSystemStreamFile(fsspec.spec.AbstractBufferedFile):
                 raise FileNotFoundError(
                     f"{e}.\nMake sure the repository and revision exist before writing data."
                 ) from e
-        # avoid an unecessary .info() call to instantiate .details
+        # avoid an unnecessary .info() call to instantiate .details
         self.details = {"name": self.resolved_path.unresolve(), "size": None}
         super().__init__(
             fs, self.resolved_path.unresolve(), mode=mode, block_size=block_size, cache_type=cache_type, **kwargs

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/hub_mixin.py

@@ -262,6 +262,12 @@ class ModelHubMixin:
         save_directory = Path(save_directory)
         save_directory.mkdir(parents=True, exist_ok=True)
 
+        # Remove config.json if already exists. After `_save_pretrained` we don't want to overwrite config.json
+        # as it might have been saved by the custom `_save_pretrained` already. However we do want to overwrite
+        # an existing config.json if it was not saved by `_save_pretrained`.
+        config_path = save_directory / CONFIG_NAME
+        config_path.unlink(missing_ok=True)
+
         # save model weights/files (framework-specific)
         self._save_pretrained(save_directory)
 
@@ -271,7 +277,6 @@ class ModelHubMixin:
         if config is not None:
             if is_dataclass(config):
                 config = asdict(config)  # type: ignore[arg-type]
-            config_path = save_directory / CONFIG_NAME
             if not config_path.exists():
                 config_str = json.dumps(config, sort_keys=True, indent=2)
                 config_path.write_text(config_str)
@@ -398,7 +403,12 @@ class ModelHubMixin:
                 # Forward config to model initialization
                 model_kwargs["config"] = config
 
-            if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()):
+            # Inject config if `**kwargs` are expected
+            if is_dataclass(cls):
+                for key in cls.__dataclass_fields__:
+                    if key not in model_kwargs and key in config:
+                        model_kwargs[key] = config[key]
+            elif any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()):
                 for key, value in config.items():
                     if key not in model_kwargs:
                         model_kwargs[key] = value

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/inference/_client.py

@@ -548,7 +548,9 @@ class InferenceClient:
         if _is_chat_completion_server(model):
             # First, let's consider the server has a `/v1/chat/completions` endpoint.
             # If that's the case, we don't have to render the chat template client-side.
-            model_url = self._resolve_url(model) + "/v1/chat/completions"
+            model_url = self._resolve_url(model)
+            if not model_url.endswith("/chat/completions"):
+                model_url += "/v1/chat/completions"
 
             try:
                 data = self.post(

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/inference/_generated/_async_client.py

@@ -548,7 +548,9 @@ class AsyncInferenceClient:
         if _is_chat_completion_server(model):
             # First, let's consider the server has a `/v1/chat/completions` endpoint.
             # If that's the case, we don't have to render the chat template client-side.
-            model_url = self._resolve_url(model) + "/v1/chat/completions"
+            model_url = self._resolve_url(model)
+            if not model_url.endswith("/chat/completions"):
+                model_url += "/v1/chat/completions"
 
             try:
                 data = await self.post(

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/lfs.py

@@ -31,6 +31,7 @@ from huggingface_hub.constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER, REPO_
 
 from .utils import (
     build_hf_headers,
+    fix_hf_endpoint_in_url,
     get_session,
     hf_raise_for_status,
     http_backoff,
@@ -193,6 +194,7 @@ def lfs_upload(
     lfs_batch_action: Dict,
     token: Optional[str] = None,
     headers: Optional[Dict[str, str]] = None,
+    endpoint: Optional[str] = None,
 ) -> None:
     """
     Handles uploading a given object to the Hub with the LFS protocol.
@@ -230,6 +232,7 @@ def lfs_upload(
     # 2. Upload file (either single part or multi-part)
     header = upload_action.get("header", {})
     chunk_size = header.get("chunk_size")
+    upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint)
     if chunk_size is not None:
         try:
             chunk_size = int(chunk_size)
@@ -237,15 +240,16 @@ def lfs_upload(
             raise ValueError(
                 f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'."
             )
-        _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_action["href"])
+        _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url)
     else:
-        _upload_single_part(operation=operation, upload_url=upload_action["href"])
+        _upload_single_part(operation=operation, upload_url=upload_url)
 
     # 3. Verify upload went well
     if verify_action is not None:
         _validate_lfs_action(verify_action)
+        verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint)
         verify_resp = get_session().post(
-            verify_action["href"],
+            verify_url,
             headers=build_hf_headers(token=token, headers=headers),
             json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size},
         )

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/utils/__init__.py

@@ -47,7 +47,14 @@ from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump
 from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential
 from ._headers import LocalTokenNotFoundError, build_hf_headers, get_token_to_send
 from ._hf_folder import HfFolder
-from ._http import OfflineModeIsEnabled, configure_http_backend, get_session, http_backoff, reset_sessions
+from ._http import (
+    OfflineModeIsEnabled,
+    configure_http_backend,
+    fix_hf_endpoint_in_url,
+    get_session,
+    http_backoff,
+    reset_sessions,
+)
 from ._pagination import paginate
 from ._paths import IGNORE_GIT_FOLDER_PATTERNS, filter_repo_objects
 from ._runtime import (

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub/utils/_http.py

@@ -21,7 +21,7 @@ import time
 import uuid
 from functools import lru_cache
 from http import HTTPStatus
-from typing import Callable, Tuple, Type, Union
+from typing import Callable, Optional, Tuple, Type, Union
 
 import requests
 from requests import Response
@@ -306,3 +306,16 @@ def http_backoff(
 
         # Update sleep time for next retry
         sleep_time = min(max_wait_time, sleep_time * 2)  # Exponential backoff
+
+
+def fix_hf_endpoint_in_url(url: str, endpoint: Optional[str]) -> str:
+    """Replace the default endpoint in a URL by a custom one.
+
+    This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint.
+    """
+    endpoint = endpoint or constants.ENDPOINT
+    # check if a proxy has been set => if yes, update the returned URL to use the proxy
+    if endpoint not in (None, constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT):
+        url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint)
+        url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint)
+    return url

{huggingface_hub-0.22.0rc0 → huggingface_hub-0.22.2}/src/huggingface_hub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: huggingface-hub
-Version: 0.22.0rc0
+Version: 0.22.2
 Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
 Home-page: https://github.com/huggingface/huggingface_hub
 Author: Hugging Face, Inc.