hfutils 0.3.7__tar.gz → 0.3.9__tar.gz

{hfutils-0.3.7/hfutils.egg-info → hfutils-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.7
+Version: 0.3.9
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992

{hfutils-0.3.7 → hfutils-0.3.9}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.3.7'
+__VERSION__ = '0.3.9'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.3.7 → hfutils-0.3.9}/hfutils/index/__init__.py

@@ -1,5 +1,6 @@
 from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
     ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch, hf_tar_file_size, hf_tar_file_info
 from .local_fetch import tar_get_index, tar_file_info, tar_file_download, tar_file_size, tar_file_exists, tar_list_files
-from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
+from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory, \
+    tar_create_index_for_directory
 from .validate import hf_tar_item_validate, hf_tar_validate

{hfutils-0.3.7 → hfutils-0.3.9}/hfutils/index/make.py

@@ -1,3 +1,22 @@
+"""
+This module provides functionalities for handling and indexing TAR archive files, especially for use with
+the Hugging Face ecosystem. It includes functions to create and retrieve index information of TAR archives,
+which is crucial for efficient data retrieval and management in large datasets. The module also integrates
+with Hugging Face's repository system, allowing for operations like uploading and downloading TAR files
+and their indices.
+
+Key functionalities include:
+
+- Extracting index information from TAR files.
+- Creating index files for TAR archives locally or in a directory.
+- Integrating with Hugging Face repositories to manage TAR archives and their indices.
+
+The module utilizes cryptographic hash functions for data integrity checks and supports operations on both local
+and remote repositories. It is designed to work seamlessly with the Hugging Face platform, enabling users to
+handle large datasets efficiently.
+"""
+
+import glob
 import json
 import logging
 import os
@@ -47,7 +66,7 @@ def tar_get_index_info(src_tar_file, chunk_for_hash: int = 1 << 20, with_hash: b
     logging.info(f'Indexing tar file {src_tar_file!r} ...')
     files = {}
     with tarfile.open(src_tar_file, mode='r|') as tar:
-        for tarinfo in tqdm(tar, desc='Indexing tar file ...', silent=silent):
+        for tarinfo in tqdm(tar, desc=f'Indexing tar file {src_tar_file!r} ...', silent=silent):
             tarinfo: tarfile.TarInfo
             if tarinfo.isreg():
                 info = {
@@ -90,11 +109,50 @@ def tar_create_index(src_tar_file, dst_index_file: Optional[str] = None,
     """
     body, _ = os.path.splitext(src_tar_file)
     dst_index_file = dst_index_file or f'{body}.json'
+    if os.path.dirname(dst_index_file):
+        os.makedirs(os.path.dirname(dst_index_file), exist_ok=True)
     with open(dst_index_file, 'w') as f:
         json.dump(tar_get_index_info(src_tar_file, chunk_for_hash, with_hash, silent), f)
     return dst_index_file
 
 
+def tar_create_index_for_directory(src_tar_directory: str, dst_index_directory: Optional[str] = None,
+                                   chunk_for_hash: int = 1 << 20, with_hash: bool = True, silent: bool = False):
+    """
+    Create index files for all tar archives in a specified directory.
+
+    This function scans through the given directory to find all tar files, generates an index for each,
+    and saves these indices to the specified destination directory. If no destination directory is provided,
+    indices are saved in the same directory as the tar files.
+
+    :param src_tar_directory: The path to the directory containing tar files.
+    :type src_tar_directory: str
+    :param dst_index_directory: The path to the directory where index files will be saved, defaults to the same as src_tar_directory.
+    :type dst_index_directory: str, optional
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param silent: Whether to suppress progress bars and logging messages, defaults to False.
+    :type silent: bool, optional
+    :return: The path to the directory where index files are saved.
+    :rtype: str
+    """
+    dst_index_directory = dst_index_directory or src_tar_directory
+    for tar_file in tqdm(glob.glob(os.path.join(src_tar_directory, '**', '*.tar'), recursive=True), silent=silent):
+        p_idx_file = os.path.join(dst_index_directory, os.path.relpath(tar_file, src_tar_directory))
+        idx_body, _ = os.path.splitext(p_idx_file)
+        idx_file = f'{idx_body}.json'
+        tar_create_index(
+            src_tar_file=tar_file,
+            dst_index_file=idx_file,
+            chunk_for_hash=chunk_for_hash,
+            with_hash=with_hash,
+            silent=silent,
+        )
+    return dst_index_directory
+
+
 def hf_tar_create_index(repo_id: str, archive_in_repo: str,
                         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                         idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,

{hfutils-0.3.7 → hfutils-0.3.9}/hfutils/operate/download.py

@@ -1,3 +1,27 @@
+"""
+This module provides functions for downloading files and directories from Hugging Face repositories.
+
+It includes utilities for downloading individual files, archives, and entire directories,
+with support for concurrent downloads, retries, and progress tracking.
+
+The module interacts with the Hugging Face Hub API to fetch repository contents and
+download files, handling various repository types and revisions.
+
+Key features:
+
+- Download individual files from Hugging Face repositories
+- Download and extract archive files
+- Download entire directories with pattern matching and ignore rules
+- Concurrent downloads with configurable worker count
+- Retry mechanism for failed downloads
+- Progress tracking with tqdm
+- Support for different repository types (dataset, model, space)
+- Token-based authentication for accessing private repositories
+
+This module is particularly useful for managing and synchronizing local copies of
+Hugging Face repository contents, especially when dealing with large datasets or models.
+"""
+
 import logging
 import os.path
 import shutil
@@ -12,9 +36,50 @@ from ..archive import archive_unpack
 from ..utils import tqdm, TemporaryDirectory, hf_normpath
 
 
+def _raw_download_file(td: str, local_file: str, repo_id: str, file_in_repo: str,
+                       repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                       hf_token: Optional[str] = None):
+    """
+    Download a file from a Hugging Face repository to a temporary directory and then move it to the final location.
+
+    This internal function handles the actual download process using the Hugging Face Hub client.
+
+    :param td: Temporary directory path.
+    :type td: str
+    :param local_file: The final local file path where the downloaded file will be moved.
+    :type local_file: str
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param file_in_repo: The file path within the repository.
+    :type file_in_repo: str
+    :param repo_type: The type of the repository ('dataset', 'model', 'space').
+    :type repo_type: RepoTypeTyping
+    :param revision: The revision of the repository (e.g., branch, tag, commit hash).
+    :type revision: str
+    :param hf_token: Hugging Face token for API client.
+    :type hf_token: str, optional
+    """
+    hf_client = get_hf_client(hf_token=hf_token)
+    relative_filename = os.path.join(*file_in_repo.split("/"))
+    temp_path = os.path.join(td, relative_filename)
+    try:
+        hf_client.hf_hub_download(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            filename=hf_normpath(file_in_repo),
+            revision=revision,
+            local_dir=td,
+        )
+    finally:
+        if os.path.exists(temp_path):
+            if os.path.dirname(local_file):
+                os.makedirs(os.path.dirname(local_file), exist_ok=True)
+            shutil.move(temp_path, local_file)
+
+
 def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
                           repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
-                          resume_download: bool = True, hf_token: Optional[str] = None):
+                          hf_token: Optional[str] = None):
     """
     Download a file from a Hugging Face repository and save it to a local file.
 
@@ -28,29 +93,19 @@ def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
     :type repo_type: RepoTypeTyping
     :param revision: The revision of the repository (e.g., branch, tag, commit hash).
     :type revision: str
-    :param resume_download: Resume the existing download.
-    :type resume_download: bool
     :param hf_token: Huggingface token for API client, use ``HF_TOKEN`` variable if not assigned.
     :type hf_token: str, optional
     """
-    hf_client = get_hf_client(hf_token)
-    relative_filename = os.path.join(*file_in_repo.split("/"))
     with TemporaryDirectory() as td:
-        temp_path = os.path.join(td, relative_filename)
-        try:
-            hf_client.hf_hub_download(
-                repo_id=repo_id,
-                repo_type=repo_type,
-                filename=hf_normpath(file_in_repo),
-                revision=revision,
-                local_dir=td,
-                resume_download=resume_download,
-            )
-        finally:
-            if os.path.exists(temp_path):
-                if os.path.dirname(local_file):
-                    os.makedirs(os.path.dirname(local_file), exist_ok=True)
-                shutil.move(temp_path, local_file)
+        _raw_download_file(
+            td=td,
+            local_file=local_file,
+            repo_id=repo_id,
+            file_in_repo=file_in_repo,
+            repo_type=repo_type,
+            revision=revision,
+            hf_token=hf_token,
+        )
 
 
 def download_archive_as_directory(local_directory: str, repo_id: str, file_in_repo: str,
@@ -84,7 +139,7 @@ def download_directory_as_directory(
         local_directory: str, repo_id: str, dir_in_repo: str = '.', pattern: str = '**/*',
         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
         silent: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET,
-        resume_download: bool = True, max_workers: int = 8, max_retries: int = 5,
+        max_workers: int = 8, max_retries: int = 5,
         soft_mode_when_check: bool = False, hf_token: Optional[str] = None
 ):
     """
@@ -110,8 +165,6 @@ def download_directory_as_directory(
     :type max_workers: int
     :param max_retries: Max retry times when downloading. Default is ``5``.
     :type max_retries: int
-    :param resume_download: Resume the existing download.
-    :type resume_download: bool
     :param soft_mode_when_check: Just check the size of the expected file when enabled. Default is False.
     :type soft_mode_when_check: bool
     :param hf_token: Huggingface token for API client, use ``HF_TOKEN`` variable if not assigned.
@@ -129,46 +182,46 @@ def download_directory_as_directory(
     progress = tqdm(files, silent=silent, desc=f'Downloading {dir_in_repo!r} ...')
 
     def _download_one_file(rel_file):
-        current_resume_download = resume_download
-        try:
-            dst_file = os.path.join(local_directory, rel_file)
-            file_in_repo = hf_normpath(f'{dir_in_repo}/{rel_file}')
-            if os.path.exists(dst_file) and is_local_file_ready(
-                    repo_id=repo_id,
-                    repo_type=repo_type,
-                    local_file=dst_file,
-                    file_in_repo=file_in_repo,
-                    revision=revision,
-                    hf_token=hf_token,
-                    soft_mode=soft_mode_when_check,
-            ):
-                logging.info(f'Local file {rel_file} is ready, download skipped.')
-            else:
-                tries = 0
-                while True:
-                    try:
-                        download_file_to_file(
-                            local_file=dst_file,
-                            repo_id=repo_id,
-                            file_in_repo=file_in_repo,
-                            repo_type=repo_type,
-                            revision=revision,
-                            resume_download=current_resume_download,
-                            hf_token=hf_token,
-                        )
-                    except requests.exceptions.RequestException as err:
-                        if tries < max_retries:
-                            tries += 1
-                            logging.warning(f'Download {rel_file!r} failed, retry ({tries}/{max_retries}) - {err!r}.')
-                            current_resume_download = True
+        with TemporaryDirectory() as td:
+            try:
+                dst_file = os.path.join(local_directory, rel_file)
+                file_in_repo = hf_normpath(f'{dir_in_repo}/{rel_file}')
+                if os.path.exists(dst_file) and is_local_file_ready(
+                        repo_id=repo_id,
+                        repo_type=repo_type,
+                        local_file=dst_file,
+                        file_in_repo=file_in_repo,
+                        revision=revision,
+                        hf_token=hf_token,
+                        soft_mode=soft_mode_when_check,
+                ):
+                    logging.info(f'Local file {rel_file} is ready, download skipped.')
+                else:
+                    tries = 0
+                    while True:
+                        try:
+                            _raw_download_file(
+                                td=td,
+                                local_file=dst_file,
+                                repo_id=repo_id,
+                                file_in_repo=file_in_repo,
+                                repo_type=repo_type,
+                                revision=revision,
+                                hf_token=hf_token,
+                            )
+                        except requests.exceptions.RequestException as err:
+                            if tries < max_retries:
+                                tries += 1
+                                logging.warning(
+                                    f'Download {rel_file!r} failed, retry ({tries}/{max_retries}) - {err!r}.')
+                            else:
+                                raise
                         else:
-                            raise
-                    else:
-                        break
+                            break
 
-            progress.update()
-        except Exception as err:
-            logging.error(f'Unexpected error when downloading {rel_file!r} - {err!r}')
+                progress.update()
+            except Exception as err:
+                logging.exception(f'Unexpected error when downloading {rel_file!r} - {err!r}')
 
     tp = ThreadPoolExecutor(max_workers=max_workers)
     for file in files:

{hfutils-0.3.7 → hfutils-0.3.9/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.7
+Version: 0.3.9
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992