huggingface-hub 0.25.0rc0__py3-none-any.whl → 0.25.1__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.25.0.rc0"
+__version__ = "0.25.1"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when

huggingface_hub/_upload_large_folder.py

@@ -41,6 +41,8 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 WAITING_TIME_IF_NO_TASKS = 10  # seconds
+MAX_NB_REGULAR_FILES_PER_COMMIT = 75
+MAX_NB_LFS_FILES_PER_COMMIT = 150
 
 
 def upload_large_folder_internal(
@@ -373,17 +375,18 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
         if (
             status.nb_workers_commit == 0
             and status.queue_commit.qsize() > 0
-            and (status.last_commit_attempt is None or time.time() - status.last_commit_attempt > 5 * 60)
+            and status.last_commit_attempt is not None
+            and time.time() - status.last_commit_attempt > 5 * 60
         ):
             status.nb_workers_commit += 1
             logger.debug("Job: commit (more than 5 minutes since last commit attempt)")
-            return (WorkerJob.COMMIT, _get_n(status.queue_commit, 25))
+            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
 
-        # 2. Commit if at least 25 files are ready to commit
-        elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 25:
+        # 2. Commit if at least 100 files are ready to commit
+        elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 150:
             status.nb_workers_commit += 1
-            logger.debug("Job: commit (>25 files ready)")
-            return (WorkerJob.COMMIT, _get_n(status.queue_commit, 25))
+            logger.debug("Job: commit (>100 files ready)")
+            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
 
         # 3. Get upload mode if at least 10 files
         elif status.queue_get_upload_mode.qsize() >= 10:
@@ -430,18 +433,39 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
             logger.debug("Job: get upload mode")
             return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, 50))
 
-        # 10. Commit if at least 1 file
-        elif status.nb_workers_commit == 0 and status.queue_commit.qsize() > 0:
+        # 10. Commit if at least 1 file and 1 min since last commit attempt
+        elif (
+            status.nb_workers_commit == 0
+            and status.queue_commit.qsize() > 0
+            and status.last_commit_attempt is not None
+            and time.time() - status.last_commit_attempt > 1 * 60
+        ):
+            status.nb_workers_commit += 1
+            logger.debug("Job: commit (1 min since last commit attempt)")
+            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
+
+        # 11. Commit if at least 1 file all other queues are empty and all workers are waiting
+        #     e.g. when it's the last commit
+        elif (
+            status.nb_workers_commit == 0
+            and status.queue_commit.qsize() > 0
+            and status.queue_sha256.qsize() == 0
+            and status.queue_get_upload_mode.qsize() == 0
+            and status.queue_preupload_lfs.qsize() == 0
+            and status.nb_workers_sha256 == 0
+            and status.nb_workers_get_upload_mode == 0
+            and status.nb_workers_preupload_lfs == 0
+        ):
             status.nb_workers_commit += 1
             logger.debug("Job: commit")
-            return (WorkerJob.COMMIT, _get_n(status.queue_commit, 25))
+            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
 
-        # 11. If all queues are empty, exit
+        # 12. If all queues are empty, exit
         elif all(metadata.is_committed or metadata.should_ignore for _, metadata in status.items):
             logger.info("All files have been processed! Exiting worker.")
             return None
 
-        # 12. If no task is available, wait
+        # 13. If no task is available, wait
         else:
             status.nb_workers_waiting += 1
             logger.debug(f"No task available, waiting... ({WAITING_TIME_IF_NO_TASKS}s)")
@@ -547,6 +571,30 @@ def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> List[JOB_ITEM_T]:
     return [queue.get() for _ in range(min(queue.qsize(), n))]
 
 
+def _get_items_to_commit(queue: "queue.Queue[JOB_ITEM_T]") -> List[JOB_ITEM_T]:
+    """Special case for commit job: the number of items to commit depends on the type of files."""
+    # Can take at most 50 regular files and/or 100 LFS files in a single commit
+    items: List[JOB_ITEM_T] = []
+    nb_lfs, nb_regular = 0, 0
+    while True:
+        # If empty queue => commit everything
+        if queue.qsize() == 0:
+            return items
+
+        # If we have enough items => commit them
+        if nb_lfs >= MAX_NB_LFS_FILES_PER_COMMIT or nb_regular >= MAX_NB_REGULAR_FILES_PER_COMMIT:
+            return items
+
+        # Else, get a new item and increase counter
+        item = queue.get()
+        items.append(item)
+        _, metadata = item
+        if metadata.upload_mode == "lfs":
+            nb_lfs += 1
+        else:
+            nb_regular += 1
+
+
 def _print_overwrite(report: str) -> None:
     """Print a report, overwriting the previous lines.
 

huggingface_hub/hf_api.py

@@ -136,6 +136,7 @@ from .utils import (
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
+from .utils._deprecation import _deprecate_method
 from .utils._typing import CallableT
 from .utils.endpoint_helpers import _is_emission_within_threshold
 
@@ -1405,6 +1406,10 @@ class User:
             Number of upvotes received by the user.
         num_likes (`int`, *optional*):
             Number of likes given by the user.
+        num_following (`int`, *optional*):
+            Number of users this user is following.
+        num_followers (`int`, *optional*):
+            Number of users following this user.
         orgs (list of [`Organization`]):
             List of organizations the user is part of.
     """
@@ -1423,6 +1428,8 @@ class User:
     num_papers: Optional[int] = None
     num_upvotes: Optional[int] = None
     num_likes: Optional[int] = None
+    num_following: Optional[int] = None
+    num_followers: Optional[int] = None
     orgs: List[Organization] = field(default_factory=list)
 
     def __init__(self, **kwargs) -> None:
@@ -1439,6 +1446,8 @@ class User:
         self.num_papers = kwargs.pop("numPapers", None)
         self.num_upvotes = kwargs.pop("numUpvotes", None)
         self.num_likes = kwargs.pop("numLikes", None)
+        self.num_following = kwargs.pop("numFollowing", None)
+        self.num_followers = kwargs.pop("numFollowers", None)
         self.user_type = kwargs.pop("type", None)
         self.orgs = [Organization(**org) for org in kwargs.pop("orgs", [])]
 
@@ -4010,6 +4019,9 @@ class HfApi:
 
     @experimental
     @validate_hf_hub_args
+    @_deprecate_method(
+        version="0.27", message="This is an experimental feature. Please use `upload_large_folder` instead."
+    )
     def create_commits_on_pr(
         self,
         *,
@@ -4848,8 +4860,10 @@ class HfApi:
                 new files. This is useful if you don't know which files have already been uploaded.
                 Note: to avoid discrepancies the `.gitattributes` file is not deleted even if it matches the pattern.
             multi_commits (`bool`):
+                Deprecated. For large uploads, use `upload_large_folder` instead.
                 If True, changes are pushed to a PR using a multi-commit process. Defaults to `False`.
             multi_commits_verbose (`bool`):
+                Deprecated. For large uploads, use `upload_large_folder` instead.
                 If True and `multi_commits` is used, more information will be displayed to the user.
             run_as_future (`bool`, *optional*):
                 Whether or not to run this method in the background. Background jobs are run sequentially without
@@ -5342,7 +5356,7 @@ class HfApi:
 
         Order of priority:
             1. Commit if more than 5 minutes since last commit attempt (and at least 1 file).
-            2. Commit if at least 25 files are ready to commit.
+            2. Commit if at least 150 files are ready to commit.
             3. Get upload mode if at least 10 files have been hashed.
             4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading.
             5. Hash file if at least 1 file and no worker is hashing.
@@ -5350,7 +5364,8 @@ class HfApi:
             7. Pre-upload LFS file if at least 1 file (exception: if hf_transfer is enabled, only 1 worker can preupload LFS at a time).
             8. Hash file if at least 1 file to hash.
             9. Get upload mode if at least 1 file to get upload mode.
-            10. Commit if at least 1 file to commit.
+            10. Commit if at least 1 file to commit and at least 1 min since last commit attempt.
+            11. Commit if at least 1 file to commit and all other queues are empty.
 
         Special rules:
             - If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`.
@@ -9463,14 +9478,24 @@ class HfApi:
                 repo_type=repo_type,
                 token=token,
             )
+        if len(filtered_repo_objects) > 30:
+            logger.info(
+                "It seems you are trying to upload a large folder at once. This might take some time and then fail if "
+                "the folder is too large. For such cases, it is recommended to upload in smaller batches or to use "
+                "`HfApi().upload_large_folder(...)`/`huggingface-cli upload-large-folder` instead. For more details, "
+                "check out https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#upload-a-large-folder."
+            )
 
-        return [
+        logger.info(f"Start hashing {len(filtered_repo_objects)} files.")
+        operations = [
             CommitOperationAdd(
                 path_or_fileobj=relpath_to_abspath[relpath],  # absolute path on disk
                 path_in_repo=prefix + relpath,  # "absolute" path in repo
             )
             for relpath in filtered_repo_objects
         ]
+        logger.info(f"Finished hashing {len(filtered_repo_objects)} files.")
+        return operations
 
     def _validate_yaml(self, content: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None):
         """

huggingface_hub/inference/_common.py

@@ -350,6 +350,12 @@ def _format_chat_completion_stream_output(
     # Decode payload
     payload = byte_payload.decode("utf-8")
     json_payload = json.loads(payload.lstrip("data:").rstrip("/n"))
+
+    # Either an error as being returned
+    if json_payload.get("error") is not None:
+        raise _parse_text_generation_error(json_payload["error"], json_payload.get("error_type"))
+
+    # Or parse token payload
     return ChatCompletionStreamOutput.parse_obj_as_instance(json_payload)
 
 

{huggingface_hub-0.25.0rc0.dist-info → huggingface_hub-0.25.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: huggingface-hub
-Version: 0.25.0rc0
+Version: 0.25.1
 Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
 Home-page: https://github.com/huggingface/huggingface_hub
 Author: Hugging Face, Inc.

{huggingface_hub-0.25.0rc0.dist-info → huggingface_hub-0.25.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-huggingface_hub/__init__.py,sha256=1dpvMU4PdU3S6Ws08T8VSpQtendIQlV8XFHg55-I_d8,34577
+huggingface_hub/__init__.py,sha256=Kq9gwfu4AHiwoP27um_dC8uc3CQGF5ldzdw6R8yMbc4,34573
 huggingface_hub/_commit_api.py,sha256=Y9eTaW4bYzxtrZsSniVtfeAuFafqx8x1ofMI5es8hvM,31057
 huggingface_hub/_commit_scheduler.py,sha256=nlJS_vnLb8i92NLrRwJX8Mg9QZ7f3kfLbLlQuEd5YjU,13647
 huggingface_hub/_inference_endpoints.py,sha256=wzjD8P68VpUDHzIDbXzFXsM2Y-aNVSAap7BXsZFuthk,16750
@@ -8,7 +8,7 @@ huggingface_hub/_multi_commits.py,sha256=mFmCP_5hNsruEgDF6kOVyaFkpnbSdNxPWfGUlFb
 huggingface_hub/_snapshot_download.py,sha256=Dbm7aPH8xrKJGE6A7aSw6YG3HgMT-VacL2s18Ut9NYA,14321
 huggingface_hub/_space_api.py,sha256=QVOUNty2T4RxPoxf9FzUjXmjHiGXP0mqXJzqQ7GmoJo,5363
 huggingface_hub/_tensorboard_logger.py,sha256=F-2l3BsHVEar8PSFhPOwprMsvkwprPgpkOTWqeMMobM,8299
-huggingface_hub/_upload_large_folder.py,sha256=_HhfBp40qXbVBR_NCctdVt_PIqzLhAFaKwPU22Y0Ei0,21534
+huggingface_hub/_upload_large_folder.py,sha256=hrm29zJXnWpxBgfJ-pmqaDwWl5WNYiYuQkKaGwXu-BA,23475
 huggingface_hub/_webhooks_payload.py,sha256=Xm3KaK7tCOGBlXkuZvbym6zjHXrT1XCrbUFWuXiBmNY,3617
 huggingface_hub/_webhooks_server.py,sha256=oCvpFrYjrhJjClAMw26SQfvN4DUItgK2IhFp1OVh2bU,15623
 huggingface_hub/community.py,sha256=4MtcoxEI9_0lmmilBEnvUEi8_O1Ivfa8p6eKxYU5-ts,12198
@@ -16,7 +16,7 @@ huggingface_hub/constants.py,sha256=BG3n2gl4JbxMw_JRvNTFyMcNnZIPzvT3KXSH-jm2J08,
 huggingface_hub/errors.py,sha256=kENdHLaBExWdu_dFJumYz7FFgLiROI4qgSvt03fJRec,9261
 huggingface_hub/fastai_utils.py,sha256=o0BnmigoNP3H7Sf8w8bwuRaN-9FKlL6uoPawr4HSyjc,16647
 huggingface_hub/file_download.py,sha256=LYbLa0OKoSeis7d3j6mB2V-IJqAkV18wYwqRSl-VTKs,85412
-huggingface_hub/hf_api.py,sha256=zeSuER_7gpe4qdxbKMTJbB0s9drQ-ut10WFmrDWVnTQ,432101
+huggingface_hub/hf_api.py,sha256=wwxfPs9a6IHZB6DOnK0hHIMcV_NQJEAM_HbFDGcxCog,433714
 huggingface_hub/hf_file_system.py,sha256=-TfO8cnfsQTxwzd-K6vjrbGkziNrAyiSHDHu_ss1U-I,39148
 huggingface_hub/hub_mixin.py,sha256=ZeCxje9e42-pm78dVhUDoQZQchXH3i1svy5CzOL3l7o,37554
 huggingface_hub/inference_api.py,sha256=b4-NhPSn9b44nYKV8tDKXodmE4JVdEymMWL4CVGkzlE,8323
@@ -41,7 +41,7 @@ huggingface_hub/commands/user.py,sha256=QApZJOCQEHADhjunM3hlQ72uqHsearCiCE4SdpzG
 huggingface_hub/commands/version.py,sha256=vfCJn7GO1m-DtDmbdsty8_RTVtnZ7lX6MJsx0Bf4e-s,1266
 huggingface_hub/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 huggingface_hub/inference/_client.py,sha256=yZ8YPWiZk0PEhNIfRsUKuoJij3QKYFqIO3jnJZa8Sho,126997
-huggingface_hub/inference/_common.py,sha256=8JNanqi7rQnFryT_11MFX9siRg-dEDlLtLwPHE6p_Rw,14906
+huggingface_hub/inference/_common.py,sha256=B-CpkIn3sBh9J2tW9rcwy1hk1Cq45pHbg4E3HZcpF6I,15121
 huggingface_hub/inference/_templating.py,sha256=xWPzz_TyQpGOZyVP_ev-gSo92swSRVK8whPAG3bbQpM,3976
 huggingface_hub/inference/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 huggingface_hub/inference/_generated/_async_client.py,sha256=_pfb8C0NV-3-ixB5bblL9-XPgjqh7ECXry7A_InEw6c,133489
@@ -109,9 +109,9 @@ huggingface_hub/utils/insecure_hashlib.py,sha256=OjxlvtSQHpbLp9PWSrXBDJ0wHjxCBU-
 huggingface_hub/utils/logging.py,sha256=Cp03s0uEl3kDM9XHQW9a8GAoExODQ-e7kEtgMt-_To8,4728
 huggingface_hub/utils/sha.py,sha256=OFnNGCba0sNcT2gUwaVCJnldxlltrHHe0DS_PCpV3C4,2134
 huggingface_hub/utils/tqdm.py,sha256=jQiVYwRG78HK4_54u0vTtz6Kt9IMGiHy3ixbIn3h2TU,9368
-huggingface_hub-0.25.0rc0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-huggingface_hub-0.25.0rc0.dist-info/METADATA,sha256=9drz2hRWlG4ta39dHHvxKI-zi8GoBvQpM0WVK0IUBpg,13057
-huggingface_hub-0.25.0rc0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-huggingface_hub-0.25.0rc0.dist-info/entry_points.txt,sha256=Y3Z2L02rBG7va_iE6RPXolIgwOdwUFONyRN3kXMxZ0g,131
-huggingface_hub-0.25.0rc0.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16
-huggingface_hub-0.25.0rc0.dist-info/RECORD,,
+huggingface_hub-0.25.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+huggingface_hub-0.25.1.dist-info/METADATA,sha256=8bSs5SXnQb--QB4tNxinaqV5niWSTVRQZgJMqn-mdkE,13054
+huggingface_hub-0.25.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+huggingface_hub-0.25.1.dist-info/entry_points.txt,sha256=Y3Z2L02rBG7va_iE6RPXolIgwOdwUFONyRN3kXMxZ0g,131
+huggingface_hub-0.25.1.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16
+huggingface_hub-0.25.1.dist-info/RECORD,,