hfutils 0.3.6__tar.gz → 0.3.8__tar.gz

{hfutils-0.3.6/hfutils.egg-info → hfutils-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.6
+Version: 0.3.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -29,6 +29,29 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
+Requires-Dist: random_user_agent
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -49,32 +72,10 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.3.6'
+__VERSION__ = '0.3.8'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/download.py

@@ -3,10 +3,12 @@ import warnings
 from typing import Optional
 
 import click
+from huggingface_hub import configure_http_backend
 
 from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
 from ..operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory
 from ..operate.base import REPO_TYPES, RepoTypeTyping
+from ..utils import get_requests_session
 
 
 class NoRemotePathAssignedWithDownload(ClickErrorException):
@@ -49,6 +51,8 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                   help='Password for the archive file. Only applied when -a is used.', show_default=True)
     @click.option('-w', '--wildcard', 'wildcard', type=str, default=None,
                   help='Wildcard for files to download. Only applied when -d is used.', show_default=True)
+    @click.option('-s', '--soft_mode_when_check', 'soft_mode_when_check', is_flag=True, type=bool, default=False,
+                  help='Just check the file size when validating the downloaded files.', show_default=True)
     @click.option('--tmpdir', 'tmpdir', type=str, default=None,
                   help='Use custom temporary Directory.', show_default=True)
     @command_wrap()
@@ -56,7 +60,7 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
             repo_id: str, repo_type: RepoTypeTyping,
             file_in_repo: Optional[str], archive_in_repo: Optional[str], dir_in_repo: Optional[str],
             output_path: str, revision: str, max_workers: int,
-            password: Optional[str], wildcard: Optional[str], tmpdir: Optional[str]
+            password: Optional[str], wildcard: Optional[str], soft_mode_when_check: bool, tmpdir: Optional[str]
     ):
         """
         Download data from HuggingFace repositories.
@@ -81,9 +85,13 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
         :type password: str, optional
         :param wildcard: Wildcard for files to download. Only applied when -d is used.
         :type password: str, optional
+        :param soft_mode_when_check: Just check the size of the expected file when enabled. Default is False.
+        :type soft_mode_when_check: bool
         :param tmpdir: Use custom temporary Directory.
         :type tmpdir: str, optional
         """
+        configure_http_backend(get_requests_session)
+
         if tmpdir:
             os.environ['TMPDIR'] = tmpdir
 
@@ -126,6 +134,7 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                 revision=revision,
                 silent=False,
                 max_workers=max_workers,
+                soft_mode_when_check=soft_mode_when_check,
             )
 
         else:

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/index.py

@@ -18,13 +18,15 @@ from typing import Optional
 
 import click
 from hbutils.string import plural_word
+from huggingface_hub import configure_http_backend
 
 from .base import CONTEXT_SETTINGS
 from ..cache import delete_detached_cache
 from ..index import hf_tar_validate, tar_create_index
 from ..operate import get_hf_fs, download_file_to_file, upload_directory_as_directory
 from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
-from ..utils import tqdm, hf_fs_path, parse_hf_fs_path, TemporaryDirectory, hf_normpath, ColoredFormatter
+from ..utils import tqdm, hf_fs_path, parse_hf_fs_path, TemporaryDirectory, hf_normpath, ColoredFormatter, \
+    get_requests_session
 
 
 def _add_index_subcommand(cli: click.Group) -> click.Group:
@@ -86,6 +88,8 @@ def _add_index_subcommand(cli: click.Group) -> click.Group:
             This function is typically invoked through the CLI interface, like:
             $ python script.py index -r my_repo -x my_index_repo -t dataset -R main --min_upload_interval 120
         """
+        configure_http_backend(get_requests_session)
+
         logger = logging.getLogger()
         logger.setLevel(logging.INFO)
         console_handler = logging.StreamHandler()

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/ls.py

@@ -5,10 +5,12 @@ from typing import Union, List
 
 import click
 import tzlocal
+from huggingface_hub import configure_http_backend
 from huggingface_hub.hf_api import RepoFolder, RepoFile
 
 from .base import CONTEXT_SETTINGS
 from ..operate.base import REPO_TYPES, get_hf_client
+from ..utils import get_requests_session
 
 mimetypes.add_type('image/webp', '.webp')
 
@@ -124,6 +126,8 @@ def _add_ls_subcommand(cli: click.Group) -> click.Group:
         :param show_detailed: Flag to indicate whether to show detailed file information.
         :type show_detailed: bool
         """
+        configure_http_backend(get_requests_session)
+
         hf_client = get_hf_client()
         items: List[ListItem] = []
         for item in hf_client.list_repo_tree(

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/ls_repo.py

@@ -2,10 +2,12 @@ import fnmatch
 from typing import Optional
 
 import click
+from huggingface_hub import configure_http_backend
 from huggingface_hub.utils import LocalTokenNotFoundError
 
 from .base import CONTEXT_SETTINGS, ClickErrorException
 from ..operate.base import REPO_TYPES, get_hf_client
+from ..utils import get_requests_session
 
 
 class NoLocalAuthentication(ClickErrorException):
@@ -46,6 +48,8 @@ def _add_ls_repo_subcommand(cli: click.Group) -> click.Group:
         :param pattern: Pattern of the repository names.
         :type pattern: str
         """
+        configure_http_backend(get_requests_session)
+
         hf_client = get_hf_client()
         if not author:
             try:

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/upload.py

@@ -2,10 +2,12 @@ import warnings
 from typing import Optional
 
 import click
+from huggingface_hub import configure_http_backend
 
 from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
 from ..operate import upload_file_to_file, upload_directory_as_archive, upload_directory_as_directory
 from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
+from ..utils import get_requests_session
 
 
 class NoRemotePathAssignedWithUpload(ClickErrorException):
@@ -78,6 +80,8 @@ def _add_upload_subcommand(cli: click.Group) -> click.Group:
         :param public: Set public repository when created.
         :type public: bool
         """
+        configure_http_backend(get_requests_session)
+
         if not file_in_repo and not archive_in_repo and not dir_in_repo:
             raise NoRemotePathAssignedWithUpload('No remote path in repository assigned.\n'
                                                  'One of the -f, -a, or -d option is required.')

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/entry/whoami.py

@@ -1,9 +1,11 @@
 import click
 from hbutils.string import plural_word
+from huggingface_hub import configure_http_backend
 from huggingface_hub.utils import LocalTokenNotFoundError
 
 from .base import CONTEXT_SETTINGS
 from ..operate.base import get_hf_client
+from ..utils import get_requests_session
 
 
 def _add_whoami_subcommand(cli: click.Group) -> click.Group:
@@ -28,6 +30,8 @@ def _add_whoami_subcommand(cli: click.Group) -> click.Group:
         This function retrieves the current user's identification from the Hugging Face Hub API and displays it.
 
         """
+        configure_http_backend(get_requests_session)
+
         hf_client = get_hf_client()
         try:
             info = hf_client.whoami()

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/index/__init__.py

@@ -1,5 +1,6 @@
 from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
     ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch, hf_tar_file_size, hf_tar_file_info
 from .local_fetch import tar_get_index, tar_file_info, tar_file_download, tar_file_size, tar_file_exists, tar_list_files
-from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
+from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory, \
+    tar_create_index_for_directory
 from .validate import hf_tar_item_validate, hf_tar_validate

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/index/make.py

@@ -1,3 +1,22 @@
+"""
+This module provides functionalities for handling and indexing TAR archive files, especially for use with
+the Hugging Face ecosystem. It includes functions to create and retrieve index information of TAR archives,
+which is crucial for efficient data retrieval and management in large datasets. The module also integrates
+with Hugging Face's repository system, allowing for operations like uploading and downloading TAR files
+and their indices.
+
+Key functionalities include:
+
+- Extracting index information from TAR files.
+- Creating index files for TAR archives locally or in a directory.
+- Integrating with Hugging Face repositories to manage TAR archives and their indices.
+
+The module utilizes cryptographic hash functions for data integrity checks and supports operations on both local
+and remote repositories. It is designed to work seamlessly with the Hugging Face platform, enabling users to
+handle large datasets efficiently.
+"""
+
+import glob
 import json
 import logging
 import os
@@ -47,7 +66,7 @@ def tar_get_index_info(src_tar_file, chunk_for_hash: int = 1 << 20, with_hash: b
     logging.info(f'Indexing tar file {src_tar_file!r} ...')
     files = {}
     with tarfile.open(src_tar_file, mode='r|') as tar:
-        for tarinfo in tqdm(tar, desc='Indexing tar file ...', silent=silent):
+        for tarinfo in tqdm(tar, desc=f'Indexing tar file {src_tar_file!r} ...', silent=silent):
             tarinfo: tarfile.TarInfo
             if tarinfo.isreg():
                 info = {
@@ -90,11 +109,50 @@ def tar_create_index(src_tar_file, dst_index_file: Optional[str] = None,
     """
     body, _ = os.path.splitext(src_tar_file)
     dst_index_file = dst_index_file or f'{body}.json'
+    if os.path.dirname(dst_index_file):
+        os.makedirs(os.path.dirname(dst_index_file), exist_ok=True)
     with open(dst_index_file, 'w') as f:
         json.dump(tar_get_index_info(src_tar_file, chunk_for_hash, with_hash, silent), f)
     return dst_index_file
 
 
+def tar_create_index_for_directory(src_tar_directory: str, dst_index_directory: Optional[str] = None,
+                                   chunk_for_hash: int = 1 << 20, with_hash: bool = True, silent: bool = False):
+    """
+    Create index files for all tar archives in a specified directory.
+
+    This function scans through the given directory to find all tar files, generates an index for each,
+    and saves these indices to the specified destination directory. If no destination directory is provided,
+    indices are saved in the same directory as the tar files.
+
+    :param src_tar_directory: The path to the directory containing tar files.
+    :type src_tar_directory: str
+    :param dst_index_directory: The path to the directory where index files will be saved, defaults to the same as src_tar_directory.
+    :type dst_index_directory: str, optional
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param silent: Whether to suppress progress bars and logging messages, defaults to False.
+    :type silent: bool, optional
+    :return: The path to the directory where index files are saved.
+    :rtype: str
+    """
+    dst_index_directory = dst_index_directory or src_tar_directory
+    for tar_file in tqdm(glob.glob(os.path.join(src_tar_directory, '**', '*.tar'), recursive=True), silent=silent):
+        p_idx_file = os.path.join(dst_index_directory, os.path.relpath(tar_file, src_tar_directory))
+        idx_body, _ = os.path.splitext(p_idx_file)
+        idx_file = f'{idx_body}.json'
+        tar_create_index(
+            src_tar_file=tar_file,
+            dst_index_file=idx_file,
+            chunk_for_hash=chunk_for_hash,
+            with_hash=with_hash,
+            silent=silent,
+        )
+    return dst_index_directory
+
+
 def hf_tar_create_index(repo_id: str, archive_in_repo: str,
                         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                         idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/operate/download.py

@@ -85,7 +85,7 @@ def download_directory_as_directory(
         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
         silent: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET,
         resume_download: bool = True, max_workers: int = 8, max_retries: int = 5,
-        hf_token: Optional[str] = None
+        soft_mode_when_check: bool = False, hf_token: Optional[str] = None
 ):
     """
     Download all files in a directory from a Hugging Face repository to a local directory.
@@ -112,6 +112,8 @@ def download_directory_as_directory(
     :type max_retries: int
     :param resume_download: Resume the existing download.
     :type resume_download: bool
+    :param soft_mode_when_check: Just check the size of the expected file when enabled. Default is False.
+    :type soft_mode_when_check: bool
     :param hf_token: Huggingface token for API client, use ``HF_TOKEN`` variable if not assigned.
     :type hf_token: str, optional
     """
@@ -138,6 +140,7 @@ def download_directory_as_directory(
                     file_in_repo=file_in_repo,
                     revision=revision,
                     hf_token=hf_token,
+                    soft_mode=soft_mode_when_check,
             ):
                 logging.info(f'Local file {rel_file} is ready, download skipped.')
             else:

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/operate/validate.py

@@ -9,7 +9,8 @@ from huggingface_hub.utils import EntryNotFoundError
 from .base import RepoTypeTyping, get_hf_client
 
 
-def _raw_check_local_file(repo_file: RepoFile, local_file: str, chunk_for_hash: int = 1 << 20) -> bool:
+def _raw_check_local_file(repo_file: RepoFile, local_file: str, chunk_for_hash: int = 1 << 20,
+                          soft_mode: bool = False) -> bool:
     """
     Checks if the local file matches the file on the Hugging Face Hub repository.
 
@@ -19,6 +20,8 @@ def _raw_check_local_file(repo_file: RepoFile, local_file: str, chunk_for_hash:
     :type local_file: str
     :param chunk_for_hash: The chunk size for calculating the hash. Default is 1 << 20.
     :type chunk_for_hash: int
+    :param soft_mode: Just check the size of the expected file when enabled. Default is False.
+    :type soft_mode: bool
     :return: True if the local file matches the file on the repository, False otherwise.
     :rtype: bool
     """
@@ -28,6 +31,9 @@ def _raw_check_local_file(repo_file: RepoFile, local_file: str, chunk_for_hash:
                       f'the remote file {repo_file.path!r} ({repo_file.size}).')
         return False
 
+    if soft_mode:
+        return True
+
     if repo_file.lfs:
         sha = sha256()
         expected_hash = repo_file.lfs.sha256
@@ -54,7 +60,7 @@ def _raw_check_local_file(repo_file: RepoFile, local_file: str, chunk_for_hash:
 
 def is_local_file_ready(local_file: str, repo_id: str, file_in_repo: str,
                         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
-                        hf_token: Optional[str] = None) -> bool:
+                        soft_mode: bool = False, hf_token: Optional[str] = None) -> bool:
     """
     Checks if the local file is ready by comparing it with the file on the Hugging Face Hub repository.
 
@@ -68,6 +74,8 @@ def is_local_file_ready(local_file: str, repo_id: str, file_in_repo: str,
     :type repo_type: RepoTypeTyping
     :param revision: The revision of the repository. Default is 'main'.
     :type revision: str
+    :param soft_mode: Just check the size of the expected file when enabled. Default is False.
+    :type soft_mode: bool
     :param hf_token: The Hugging Face API token. Default is None.
     :type hf_token: Optional[str]
     :return: True if the local file matches the file on the repository, False otherwise.
@@ -83,6 +91,6 @@ def is_local_file_ready(local_file: str, repo_id: str, file_in_repo: str,
     if len(infos) == 0:
         raise EntryNotFoundError(f'Entry {repo_type}s/{repo_id}/{file_in_repo} not found.')
     elif len(infos) == 1:
-        return _raw_check_local_file(infos[0], local_file)
+        return _raw_check_local_file(infos[0], local_file, soft_mode=soft_mode)
     else:
         assert False, f'Should not reach here, multiple files with the same name found - {infos!r}'  # pragma: no cover

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils/utils/__init__.py

@@ -3,6 +3,7 @@ from .download import download_file
 from .logging import ColoredFormatter
 from .number import number_to_tag
 from .path import hf_normpath, hf_fs_path, parse_hf_fs_path, HfFileSystemPath
+from .session import TimeoutHTTPAdapter, get_requests_session, get_random_ua
 from .temp import TemporaryDirectory
 from .tqdm_ import tqdm
 from .walk import walk_files

hfutils-0.3.8/hfutils/utils/session.py

@@ -0,0 +1,115 @@
+"""
+This module provides functionality for creating and managing HTTP sessions with customizable retry logic,
+timeout settings, and user-agent rotation using random user-agent generation. It is designed to help with
+robust web scraping and API consumption by handling common HTTP errors and timeouts gracefully.
+
+Main Features:
+
+- Automatic retries on specified HTTP response status codes.
+- Configurable request timeout.
+- Rotating user-agent for each session to mimic different browsers and operating systems.
+- Optional SSL verification.
+"""
+
+from functools import lru_cache
+from typing import Optional, Dict
+
+import requests
+from random_user_agent.params import SoftwareName, OperatingSystem
+from random_user_agent.user_agent import UserAgent
+from requests.adapters import HTTPAdapter, Retry
+
+DEFAULT_TIMEOUT = 15  # seconds
+
+
+class TimeoutHTTPAdapter(HTTPAdapter):
+    """
+    A custom HTTPAdapter that enforces a default timeout on all requests.
+
+    :param args: Variable length argument list for HTTPAdapter.
+    :param kwargs: Arbitrary keyword arguments. 'timeout' can be specified to set a custom timeout.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.timeout = DEFAULT_TIMEOUT
+        if "timeout" in kwargs:
+            self.timeout = kwargs["timeout"]
+            del kwargs["timeout"]
+        super().__init__(*args, **kwargs)
+
+    def send(self, request, **kwargs):
+        """
+        Sends the Request object, applying the timeout setting.
+
+        :param request: The Request object to send.
+        :type request: requests.PreparedRequest
+        :param kwargs: Keyword arguments that may contain 'timeout'.
+        :return: The response to the request.
+        """
+        timeout = kwargs.get("timeout")
+        if timeout is None:
+            kwargs["timeout"] = self.timeout
+        return super().send(request, **kwargs)
+
+
+def get_requests_session(max_retries: int = 5, timeout: int = DEFAULT_TIMEOUT, verify: bool = True,
+                         headers: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None) \
+        -> requests.Session:
+    """
+    Creates a requests session with retry logic, timeout settings, and random user-agent headers.
+
+    :param max_retries: Maximum number of retries on failed requests.
+    :type max_retries: int
+    :param timeout: Request timeout in seconds.
+    :type timeout: int
+    :param verify: Whether to verify SSL certificates.
+    :type verify: bool
+    :param headers: Additional headers to include in the requests.
+    :type headers: Optional[Dict[str, str]]
+    :param session: An existing requests.Session instance to use.
+    :type session: Optional[requests.Session]
+    :return: A configured requests.Session object.
+    :rtype: requests.Session
+    """
+    session = session or requests.session()
+    retries = Retry(
+        total=max_retries, backoff_factor=1,
+        status_forcelist=[408, 429, 500, 501, 502, 503, 504, 505, 506, 507, 509, 510, 511],
+        allowed_methods=["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"],
+    )
+    adapter = TimeoutHTTPAdapter(max_retries=retries, timeout=timeout, pool_connections=32, pool_maxsize=32)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    session.headers.update({
+        "User-Agent": get_random_ua(),
+        **dict(headers or {}),
+    })
+    if not verify:
+        session.verify = False
+
+    return session
+
+
+@lru_cache()
+def _ua_pool():
+    """
+    Creates and caches a UserAgent rotator instance with a specified number of user agents.
+
+    :return: A UserAgent rotator instance.
+    :rtype: UserAgent
+    """
+    software_names = [SoftwareName.CHROME.value, SoftwareName.FIREFOX.value, SoftwareName.EDGE.value]
+    operating_systems = [OperatingSystem.WINDOWS.value, OperatingSystem.MACOS.value]
+
+    user_agent_rotator = UserAgent(software_names=software_names, operating_systems=operating_systems, limit=1000)
+    return user_agent_rotator
+
+
+def get_random_ua():
+    """
+    Retrieves a random user agent string from the cached UserAgent rotator.
+
+    :return: A random user agent string.
+    :rtype: str
+    """
+    return _ua_pool().get_random_user_agent()

{hfutils-0.3.6 → hfutils-0.3.8/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.6
+Version: 0.3.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -29,6 +29,29 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
+Requires-Dist: random_user_agent
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -49,32 +72,10 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils.egg-info/SOURCES.txt

@@ -61,6 +61,7 @@ hfutils/utils/irregular_repo.json
 hfutils/utils/logging.py
 hfutils/utils/number.py
 hfutils/utils/path.py
+hfutils/utils/session.py
 hfutils/utils/temp.py
 hfutils/utils/tqdm_.py
 hfutils/utils/walk.py

{hfutils-0.3.6 → hfutils-0.3.8}/hfutils.egg-info/requires.txt

@@ -7,6 +7,7 @@ tzlocal
 natsort
 urlobject
 fsspec>=2024
+random_user_agent
 
 [7z]
 py7zr

{hfutils-0.3.6 → hfutils-0.3.8}/requirements.txt

@@ -6,4 +6,5 @@ click>=7
 tzlocal
 natsort
 urlobject
-fsspec>=2024
+fsspec>=2024
+random_user_agent