hfutils 0.4.1__tar.gz → 0.4.3__tar.gz

{hfutils-0.4.1/hfutils.egg-info → hfutils-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.4.1
+Version: 0.4.3
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,28 +30,13 @@ Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
 Requires-Dist: random_user_agent
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -72,10 +57,25 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 
 # hfutils
 

{hfutils-0.4.1 → hfutils-0.4.3}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.4.1'
+__VERSION__ = '0.4.3'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.4.1 → hfutils-0.4.3}/hfutils/index/fetch.py

@@ -6,6 +6,7 @@ from typing import Optional, Dict, Union, List
 
 from huggingface_hub.file_download import http_get, hf_hub_url
 from huggingface_hub.utils import build_hf_headers
+from tqdm import tqdm
 
 from .hash import _f_sha256
 from ..operate.base import RepoTypeTyping, get_hf_client
@@ -438,23 +439,27 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                          idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
                          proxies: Optional[Dict] = None, user_agent: Union[Dict, str, None] = None,
                          headers: Optional[Dict[str, str]] = None, endpoint: Optional[str] = None,
-                         hf_token: Optional[str] = None):
+                         force_download: bool = False, silent: bool = False, hf_token: Optional[str] = None):
     """
-    Download a file from a tar archive file in a Hugging Face repository.
+    Download a specific file from a tar archive stored in a Hugging Face repository.
 
-    :param repo_id: The identifier of the repository.
+    This function allows you to extract and download a single file from a tar archive
+    that is hosted in a Hugging Face repository. It handles authentication, supports
+    different repository types, and can work with separate index repositories.
+
+    :param repo_id: The identifier of the repository containing the tar archive.
     :type repo_id: str
-    :param archive_in_repo: The path to the archive file in the repository.
+    :param archive_in_repo: The path to the tar archive file within the repository.
     :type archive_in_repo: str
-    :param file_in_archive: The path to the file inside the archive.
+    :param file_in_archive: The path to the desired file inside the tar archive.
     :type file_in_archive: str
-    :param local_file: The path to save the downloaded file locally.
+    :param local_file: The local path where the downloaded file will be saved.
     :type local_file: str
-    :param repo_type: The type of the Hugging Face repository.
+    :param repo_type: The type of the Hugging Face repository (e.g., 'dataset', 'model', 'space').
     :type repo_type: RepoTypeTyping, optional
-    :param revision: The revision of the repository.
+    :param revision: The specific revision of the repository to use.
     :type revision: str, optional
-    :param idx_repo_id: The identifier of the index repository.
+    :param idx_repo_id: The identifier of a separate index repository, if applicable.
     :type idx_repo_id: str, optional
     :param idx_file_in_repo: The path to the index file in the index repository.
     :type idx_file_in_repo: str, optional
@@ -462,44 +467,58 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     :type idx_repo_type: RepoTypeTyping, optional
     :param idx_revision: The revision of the index repository.
     :type idx_revision: str, optional
-    :param proxies: The proxies to be used for the HTTP request.
+    :param proxies: Proxy settings for the HTTP request.
     :type proxies: Dict, optional
-    :param user_agent: The user agent for the HTTP request.
+    :param user_agent: Custom user agent for the HTTP request.
     :type user_agent: Union[Dict, str, None], optional
-    :param headers: The additional headers for the HTTP request.
+    :param headers: Additional headers for the HTTP request.
     :type headers: Dict[str, str], optional
-    :param endpoint: The Hugging Face API endpoint.
+    :param endpoint: Custom Hugging Face API endpoint.
     :type endpoint: str, optional
-    :param hf_token: The Hugging Face access token.
+    :param force_download: If True, force re-download even if the file exists locally.
+    :type force_download: bool
+    :param silent: If True, suppress progress bar output.
+    :type silent: bool
+    :param hf_token: Hugging Face authentication token.
     :type hf_token: str, optional
-    :raises FileNotFoundError: Raise this when file not exist in tar archive.
-    :raises ArchiveStandaloneFileIncompleteDownload: Raise when download incomplete.
-    :raises ArchiveStandaloneFileHashNotMatch: Raise when download hash not match.
 
-    Examples::
-        >>> from hfutils.index import hf_tar_file_download
-        >>>
-        >>> hf_tar_file_download(
-        ...     repo_id='deepghs/danbooru_newest',
-        ...     archive_in_repo='images/0000.tar',
-        ...     file_in_archive='7506000.jpg',
-        ...     local_file='test_example.jpg'  # download destination
-        ... )
+    :raises FileNotFoundError: If the specified file is not found in the tar archive.
+    :raises ArchiveStandaloneFileIncompleteDownload: If the download is incomplete.
+    :raises ArchiveStandaloneFileHashNotMatch: If the downloaded file's hash doesn't match the expected hash.
+
+    This function performs several steps:
+
+    1. Retrieves the index of the tar archive.
+    2. Checks if the desired file exists in the archive.
+    3. Constructs the download URL and headers.
+    4. Checks if the file already exists locally and matches the expected size and hash.
+    5. Downloads the file if necessary, using byte range requests for efficiency.
+    6. Verifies the downloaded file's size and hash.
+
+    Usage examples:
+        1. Basic usage:
+            >>> hf_tar_file_download(
+            ...     repo_id='deepghs/danbooru_newest',
+            ...     archive_in_repo='images/0000.tar',
+            ...     file_in_archive='7506000.jpg',
+            ...     local_file='test_example.jpg'  # download destination
+            ... )
+
+        2. Using a separate index repository:
+            >>> hf_tar_file_download(
+            ...     repo_id='nyanko7/danbooru2023',
+            ...     idx_repo_id='deepghs/danbooru2023_index',
+            ...     archive_in_repo='original/data-0000.tar',
+            ...     file_in_archive='1000.png',
+            ...     local_file='test_example.png'  # download destination
+            ... )
 
     .. note::
 
-        Besides, if the tar and index files are in different repositories, you can also use this function to
-        download the given file by explicitly assigning the ``idx_repo_id`` argument.
-
-        >>> from hfutils.index import hf_tar_file_download
-        >>>
-        >>> hf_tar_file_download(
-        ...     repo_id='nyanko7/danbooru2023',
-        ...     idx_repo_id='deepghs/danbooru2023_index',
-        ...     archive_in_repo='original/data-0000.tar',
-        ...     file_in_archive='1000.png',
-        ...     local_file='test_example.png'  # download destination
-        ... )
+        - This function is particularly useful for efficiently downloading single files from large tar archives
+          without having to download the entire archive.
+        - It supports authentication via the `hf_token` parameter, which is crucial for accessing private repositories.
+        - The function includes checks to avoid unnecessary downloads and to ensure the integrity of the downloaded file.
     """
     index = hf_tar_get_index(
         repo_id=repo_id,
@@ -533,10 +552,17 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     end_bytes = info['offset'] + info['size'] - 1
     headers['Range'] = f'bytes={start_bytes}-{end_bytes}'
 
+    if not force_download and os.path.exists(local_file) and \
+            os.path.isfile(local_file) and os.path.getsize(local_file) == info['size']:
+        _expected_sha256 = info.get('sha256')
+        if not _expected_sha256 or _f_sha256(local_file) == _expected_sha256:
+            # file already ready, no need to download it again
+            return
+
     if os.path.dirname(local_file):
         os.makedirs(os.path.dirname(local_file), exist_ok=True)
     try:
-        with open(local_file, 'wb') as f:
+        with open(local_file, 'wb') as f, tqdm(disable=True) as empty_tqdm:
             if info['size'] > 0:
                 http_get(
                     url_to_download,
@@ -546,6 +572,7 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                     headers=headers,
                     expected_size=info['size'],
                     displayed_filename=file_in_archive,
+                    _tqdm_bar=empty_tqdm if silent else None,
                 )
 
         if os.path.getsize(local_file) != info['size']:

{hfutils-0.4.1 → hfutils-0.4.3}/hfutils/index/local_fetch.py

@@ -171,7 +171,7 @@ def tar_file_size(archive_file: str, file_in_archive: str, idx_file: Optional[st
 
 
 def tar_file_download(archive_file: str, file_in_archive: str, local_file: str,
-                      idx_file: Optional[str] = None, chunk_size: int = 1 << 20):
+                      idx_file: Optional[str] = None, chunk_size: int = 1 << 20, force_download: bool = False):
     """
     Extract and download a specific file from the tar archive to a local file.
 
@@ -190,6 +190,10 @@ def tar_file_download(archive_file: str, file_in_archive: str, local_file: str,
     :type idx_file: Optional[str]
     :param chunk_size: The size of chunks to read and write, in bytes. Default is 1MB.
     :type chunk_size: int
+    :param force_download: Force download the file to destination path.
+                           Defualt to `False`, downloading will be skipped if the local file
+                           is fully matched with expected file.
+    :type force_download: bool
 
     :raises FileNotFoundError: If the specified file is not found in the archive.
     :raises ArchiveStandaloneFileIncompleteDownload: If the downloaded file size doesn't match the expected size.
@@ -212,6 +216,13 @@ def tar_file_download(archive_file: str, file_in_archive: str, local_file: str,
 
     info = files[_n_path(file_in_archive)]
 
+    if not force_download and os.path.exists(local_file) and \
+            os.path.isfile(local_file) and os.path.getsize(local_file) == info['size']:
+        _expected_sha256 = info.get('sha256')
+        if not _expected_sha256 or _f_sha256(local_file) == _expected_sha256:
+            # file already ready, no need to download it again
+            return
+
     if os.path.dirname(local_file):
         os.makedirs(os.path.dirname(local_file), exist_ok=True)
     try:

{hfutils-0.4.1 → hfutils-0.4.3/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.4.1
+Version: 0.4.3
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,28 +30,13 @@ Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
 Requires-Dist: random_user_agent
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -72,10 +57,25 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 
 # hfutils