hfutils 0.4.2__tar.gz → 0.4.4__tar.gz

{hfutils-0.4.2/hfutils.egg-info → hfutils-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.4.2
+Version: 0.4.4
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,28 +30,13 @@ Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
 Requires-Dist: random_user_agent
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -72,10 +57,25 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 
 # hfutils
 

{hfutils-0.4.2 → hfutils-0.4.4}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.4.2'
+__VERSION__ = '0.4.4'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.4.2 → hfutils-0.4.4}/hfutils/index/fetch.py

@@ -6,6 +6,7 @@ from typing import Optional, Dict, Union, List
 
 from huggingface_hub.file_download import http_get, hf_hub_url
 from huggingface_hub.utils import build_hf_headers
+from tqdm import tqdm
 
 from .hash import _f_sha256
 from ..operate.base import RepoTypeTyping, get_hf_client
@@ -438,23 +439,27 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                          idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
                          proxies: Optional[Dict] = None, user_agent: Union[Dict, str, None] = None,
                          headers: Optional[Dict[str, str]] = None, endpoint: Optional[str] = None,
-                         force_download: bool = False, hf_token: Optional[str] = None):
+                         force_download: bool = False, silent: bool = False, hf_token: Optional[str] = None):
     """
-    Download a file from a tar archive file in a Hugging Face repository.
+    Download a specific file from a tar archive stored in a Hugging Face repository.
 
-    :param repo_id: The identifier of the repository.
+    This function allows you to extract and download a single file from a tar archive
+    that is hosted in a Hugging Face repository. It handles authentication, supports
+    different repository types, and can work with separate index repositories.
+
+    :param repo_id: The identifier of the repository containing the tar archive.
     :type repo_id: str
-    :param archive_in_repo: The path to the archive file in the repository.
+    :param archive_in_repo: The path to the tar archive file within the repository.
     :type archive_in_repo: str
-    :param file_in_archive: The path to the file inside the archive.
+    :param file_in_archive: The path to the desired file inside the tar archive.
     :type file_in_archive: str
-    :param local_file: The path to save the downloaded file locally.
+    :param local_file: The local path where the downloaded file will be saved.
     :type local_file: str
-    :param repo_type: The type of the Hugging Face repository.
+    :param repo_type: The type of the Hugging Face repository (e.g., 'dataset', 'model', 'space').
     :type repo_type: RepoTypeTyping, optional
-    :param revision: The revision of the repository.
+    :param revision: The specific revision of the repository to use.
     :type revision: str, optional
-    :param idx_repo_id: The identifier of the index repository.
+    :param idx_repo_id: The identifier of a separate index repository, if applicable.
     :type idx_repo_id: str, optional
     :param idx_file_in_repo: The path to the index file in the index repository.
     :type idx_file_in_repo: str, optional
@@ -462,48 +467,58 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     :type idx_repo_type: RepoTypeTyping, optional
     :param idx_revision: The revision of the index repository.
     :type idx_revision: str, optional
-    :param proxies: The proxies to be used for the HTTP request.
+    :param proxies: Proxy settings for the HTTP request.
     :type proxies: Dict, optional
-    :param user_agent: The user agent for the HTTP request.
+    :param user_agent: Custom user agent for the HTTP request.
     :type user_agent: Union[Dict, str, None], optional
-    :param headers: The additional headers for the HTTP request.
+    :param headers: Additional headers for the HTTP request.
     :type headers: Dict[str, str], optional
-    :param endpoint: The Hugging Face API endpoint.
+    :param endpoint: Custom Hugging Face API endpoint.
     :type endpoint: str, optional
-    :param force_download: Force download the file to destination path.
-                           Defualt to `False`, downloading will be skipped if the local file
-                           is fully matched with expected file.
+    :param force_download: If True, force re-download even if the file exists locally.
     :type force_download: bool
-    :param hf_token: The Hugging Face access token.
+    :param silent: If True, suppress progress bar output.
+    :type silent: bool
+    :param hf_token: Hugging Face authentication token.
     :type hf_token: str, optional
-    :raises FileNotFoundError: Raise this when file not exist in tar archive.
-    :raises ArchiveStandaloneFileIncompleteDownload: Raise when download incomplete.
-    :raises ArchiveStandaloneFileHashNotMatch: Raise when download hash not match.
 
-    Examples::
-        >>> from hfutils.index import hf_tar_file_download
-        >>>
-        >>> hf_tar_file_download(
-        ...     repo_id='deepghs/danbooru_newest',
-        ...     archive_in_repo='images/0000.tar',
-        ...     file_in_archive='7506000.jpg',
-        ...     local_file='test_example.jpg'  # download destination
-        ... )
+    :raises FileNotFoundError: If the specified file is not found in the tar archive.
+    :raises ArchiveStandaloneFileIncompleteDownload: If the download is incomplete.
+    :raises ArchiveStandaloneFileHashNotMatch: If the downloaded file's hash doesn't match the expected hash.
+
+    This function performs several steps:
+
+    1. Retrieves the index of the tar archive.
+    2. Checks if the desired file exists in the archive.
+    3. Constructs the download URL and headers.
+    4. Checks if the file already exists locally and matches the expected size and hash.
+    5. Downloads the file if necessary, using byte range requests for efficiency.
+    6. Verifies the downloaded file's size and hash.
+
+    Usage examples:
+        1. Basic usage:
+            >>> hf_tar_file_download(
+            ...     repo_id='deepghs/danbooru_newest',
+            ...     archive_in_repo='images/0000.tar',
+            ...     file_in_archive='7506000.jpg',
+            ...     local_file='test_example.jpg'  # download destination
+            ... )
+
+        2. Using a separate index repository:
+            >>> hf_tar_file_download(
+            ...     repo_id='nyanko7/danbooru2023',
+            ...     idx_repo_id='deepghs/danbooru2023_index',
+            ...     archive_in_repo='original/data-0000.tar',
+            ...     file_in_archive='1000.png',
+            ...     local_file='test_example.png'  # download destination
+            ... )
 
     .. note::
 
-        Besides, if the tar and index files are in different repositories, you can also use this function to
-        download the given file by explicitly assigning the ``idx_repo_id`` argument.
-
-        >>> from hfutils.index import hf_tar_file_download
-        >>>
-        >>> hf_tar_file_download(
-        ...     repo_id='nyanko7/danbooru2023',
-        ...     idx_repo_id='deepghs/danbooru2023_index',
-        ...     archive_in_repo='original/data-0000.tar',
-        ...     file_in_archive='1000.png',
-        ...     local_file='test_example.png'  # download destination
-        ... )
+        - This function is particularly useful for efficiently downloading single files from large tar archives
+          without having to download the entire archive.
+        - It supports authentication via the `hf_token` parameter, which is crucial for accessing private repositories.
+        - The function includes checks to avoid unnecessary downloads and to ensure the integrity of the downloaded file.
     """
     index = hf_tar_get_index(
         repo_id=repo_id,
@@ -547,7 +562,7 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     if os.path.dirname(local_file):
         os.makedirs(os.path.dirname(local_file), exist_ok=True)
     try:
-        with open(local_file, 'wb') as f:
+        with open(local_file, 'wb') as f, tqdm(disable=True) as empty_tqdm:
             if info['size'] > 0:
                 http_get(
                     url_to_download,
@@ -557,6 +572,7 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                     headers=headers,
                     expected_size=info['size'],
                     displayed_filename=file_in_archive,
+                    _tqdm_bar=empty_tqdm if silent else None,
                 )
 
         if os.path.getsize(local_file) != info['size']:

{hfutils-0.4.2 → hfutils-0.4.4}/hfutils/repository/__init__.py

@@ -5,3 +5,4 @@ Overview:
 from .base import hf_hub_repo_url
 from .clone import hf_hub_clone
 from .rollback import hf_hub_rollback
+from .url import hf_hub_repo_path_url, hf_hub_repo_file_url

hfutils-0.4.4/hfutils/repository/url.py

@@ -0,0 +1,86 @@
+"""
+This module provides functions for generating URLs related to Hugging Face Hub repositories.
+
+It includes functions to create URLs for repository paths and files, which can be useful
+for accessing and referencing resources within Hugging Face Hub repositories.
+"""
+
+from typing import Optional
+from urllib.parse import quote_plus, urljoin
+
+from .base import hf_hub_repo_url
+from ..utils.path import RepoTypeTyping, hf_normpath
+
+
+def hf_hub_repo_path_url(repo_id: str, path: str, repo_type: RepoTypeTyping = 'dataset',
+                         revision: str = 'main', endpoint: Optional[str] = None) -> str:
+    """
+    Generate the URL for a path within a Hugging Face Hub repository.
+
+    This function constructs a URL that points to a specific path within a Hugging Face Hub repository.
+    It's useful for creating links to directories or files in the repository's web interface.
+
+    :param repo_id: The repository ID (e.g., 'username/repo-name').
+    :type repo_id: str
+    :param path: The path within the repository to generate the URL for.
+    :type path: str
+    :param repo_type: The type of the repository. Defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision (branch, tag, or commit) to use. Defaults to 'main'.
+    :type revision: str, optional
+    :param endpoint: The API endpoint to use. If None, the default endpoint will be used.
+    :type endpoint: str, optional
+
+    :return: The URL for the specified path in the Hugging Face Hub repository.
+    :rtype: str
+
+    :example:
+        >>> url = hf_hub_repo_path_url('username/my-dataset', 'data/train.csv')
+        >>> print(url)
+        https://huggingface.co/datasets/username/my-dataset/tree/main/data/train.csv
+    """
+    repo_url = hf_hub_repo_url(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        endpoint=endpoint,
+    )
+    return urljoin(repo_url + '/', hf_normpath(f'tree/{quote_plus(revision)}/{hf_normpath(path)}'))
+
+
+def hf_hub_repo_file_url(repo_id: str, path: str, repo_type: RepoTypeTyping = 'dataset',
+                         revision: str = 'main', endpoint: Optional[str] = None) -> str:
+    """
+    Generate the URL for a file within a Hugging Face Hub repository.
+
+    This function constructs a URL that points to a specific file within a Hugging Face Hub repository.
+    It's useful for creating links to view files in the repository's web interface.
+
+    :param repo_id: The repository ID (e.g., 'username/repo-name').
+    :type repo_id: str
+    :param path: The path to the file within the repository.
+    :type path: str
+    :param repo_type: The type of the repository. Defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision (branch, tag, or commit) to use. Defaults to 'main'.
+    :type revision: str, optional
+    :param endpoint: The API endpoint to use. If None, the default endpoint will be used.
+    :type endpoint: str, optional
+
+    :return: The URL for the specified file in the Hugging Face Hub repository.
+    :rtype: str
+
+    :example:
+        >>> url = hf_hub_repo_file_url('username/my-model', 'config.json')
+        >>> print(url)
+        https://huggingface.co/username/my-model/blob/main/config.json
+
+    .. note::
+        This is not the downloadable URL of the file in the repository.
+        For downloading URLs, use :func:`huggingface_hub.hf_hub_url`.
+    """
+    repo_url = hf_hub_repo_url(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        endpoint=endpoint,
+    )
+    return urljoin(repo_url + '/', hf_normpath(f'blob/{quote_plus(revision)}/{hf_normpath(path)}'))

{hfutils-0.4.2 → hfutils-0.4.4/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.4.2
+Version: 0.4.4
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,28 +30,13 @@ Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
 Requires-Dist: random_user_agent
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -72,10 +57,25 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 
 # hfutils
 

{hfutils-0.4.2 → hfutils-0.4.4}/hfutils.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ hfutils/repository/__init__.py
 hfutils/repository/base.py
 hfutils/repository/clone.py
 hfutils/repository/rollback.py
+hfutils/repository/url.py
 hfutils/utils/__init__.py
 hfutils/utils/archive.py
 hfutils/utils/binary.py