hfutils 0.2.0__tar.gz → 0.2.2__tar.gz

{hfutils-0.2.0/hfutils.egg-info → hfutils-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.0
+Version: 0.2.2
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992

{hfutils-0.2.0 → hfutils-0.2.2}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.0'
+__VERSION__ = '0.2.2'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.0 → hfutils-0.2.2}/hfutils/index/__init__.py

@@ -1,3 +1,4 @@
 from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
     ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch
 from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
+from .validate import hf_tar_item_validate, hf_tar_validate

{hfutils-0.2.0 → hfutils-0.2.2}/hfutils/index/fetch.py

@@ -240,6 +240,7 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
                                 f'in {repo_type}s/{repo_id}@{revision}/{archive_in_repo}.')
 
     info = files[_n_path(file_in_archive)]
+
     url_to_download = hf_hub_url(repo_id, archive_in_repo, repo_type=repo_type, revision=revision, endpoint=endpoint)
     headers = build_hf_headers(
         token=hf_token,
@@ -256,15 +257,16 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
         os.makedirs(os.path.dirname(local_file), exist_ok=True)
     try:
         with open(local_file, 'wb') as f:
-            http_get(
-                url_to_download,
-                f,
-                proxies=proxies,
-                resume_size=0,
-                headers=headers,
-                expected_size=info['size'],
-                displayed_filename=file_in_archive,
-            )
+            if info['size'] > 0:
+                http_get(
+                    url_to_download,
+                    f,
+                    proxies=proxies,
+                    resume_size=0,
+                    headers=headers,
+                    expected_size=info['size'],
+                    displayed_filename=file_in_archive,
+                )
 
         if os.path.getsize(local_file) != info['size']:
             raise ArchiveStandaloneFileIncompleteDownload(

{hfutils-0.2.0 → hfutils-0.2.2}/hfutils/index/make.py

@@ -92,25 +92,27 @@ def tar_create_index(src_tar_file, dst_index_file: Optional[str] = None,
     return dst_index_file
 
 
-def hf_tar_create_index(repo_id: str, filename: str, repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
-                        idx_repo_id: Optional[str] = None, idx_filename: Optional[str] = None,
+def hf_tar_create_index(repo_id: str, archive_in_repo: str,
+                        repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                        idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
                         idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
-                        chunk_for_hash: int = 1 << 20, with_hash: bool = True, hf_token: Optional[str] = None):
+                        chunk_for_hash: int = 1 << 20, with_hash: bool = True, skip_when_synced: bool = True,
+                        hf_token: Optional[str] = None, ):
     """
     Create an index file for a tar archive file in a Hugging Face repository.
 
     :param repo_id: The identifier of the repository.
     :type repo_id: str
-    :param filename: The path to the tar archive file.
-    :type filename: str
+    :param archive_in_repo: The path to the tar archive file.
+    :type archive_in_repo: str
     :param repo_type: The type of the Hugging Face repository, defaults to 'dataset'.
     :type repo_type: RepoTypeTyping, optional
     :param revision: The revision of the repository, defaults to 'main'.
     :type revision: str, optional
     :param idx_repo_id: The identifier of the index repository, defaults to None.
     :type idx_repo_id: str, optional
-    :param idx_filename: The path to save the index file in the index repository, defaults to None.
-    :type idx_filename: str, optional
+    :param idx_file_in_repo: The path to save the index file in the index repository, defaults to None.
+    :type idx_file_in_repo: str, optional
     :param idx_repo_type: The type of the index repository, defaults to None.
     :type idx_repo_type: RepoTypeTyping, optional
     :param idx_revision: The revision of the index repository, defaults to None.
@@ -119,31 +121,51 @@ def hf_tar_create_index(repo_id: str, filename: str, repo_type: RepoTypeTyping =
     :type chunk_for_hash: int, optional
     :param with_hash: Whether to include file hashes in the index, defaults to True.
     :type with_hash: bool, optional
+    :param skip_when_synced: Skip syncing when index is ready, defaults to True.
+    :type skip_when_synced: bool
     :param hf_token: The Hugging Face access token, defaults to None.
     :type hf_token: str, optional
     """
+    body, _ = os.path.splitext(archive_in_repo)
+    default_index_filename = f'{body}.json'
+
+    from .validate import hf_tar_validate
+    if skip_when_synced and hf_tar_validate(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            archive_in_repo=archive_in_repo,
+            revision=revision,
+
+            idx_repo_id=idx_repo_id or repo_id,
+            idx_repo_type=idx_repo_type or repo_type,
+            idx_file_in_repo=idx_file_in_repo or default_index_filename,
+            idx_revision=idx_revision or revision,
+
+            hf_token=hf_token,
+    ):
+        logging.info(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} already indexed, skipped.')
+        return
+
     with TemporaryDirectory() as td:
-        local_tar_file = os.path.join(td, os.path.basename(filename))
+        local_tar_file = os.path.join(td, os.path.basename(archive_in_repo))
         download_file_to_file(
             repo_id=repo_id,
             repo_type=repo_type,
-            file_in_repo=filename,
+            file_in_repo=archive_in_repo,
             local_file=local_tar_file,
             revision=revision,
             hf_token=hf_token,
         )
         dst_index_file = tar_create_index(local_tar_file, chunk_for_hash=chunk_for_hash, with_hash=with_hash)
 
-        body, _ = os.path.splitext(filename)
-        default_index_filename = f'{body}.json'
         upload_file_to_file(
             repo_id=idx_repo_id or repo_id,
             repo_type=idx_repo_type or repo_type,
-            file_in_repo=idx_filename or default_index_filename,
+            file_in_repo=idx_file_in_repo or default_index_filename,
             local_file=dst_index_file,
             revision=idx_revision or revision,
             hf_token=hf_token,
-            message=f'Create index for {repo_type}s/{repo_id}@{revision}/{filename}',
+            message=f'Create index for {repo_type}s/{repo_id}@{revision}/{archive_in_repo}',
         )
 
 

hfutils-0.2.2/hfutils/index/validate.py

@@ -0,0 +1,111 @@
+from typing import Optional
+
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
+
+from .fetch import hf_tar_get_index
+from ..operate.base import RepoTypeTyping, get_hf_client
+
+
+def hf_tar_item_validate(file_item: RepoFile, size: int, hash_: Optional[str] = None, hash_lfs: Optional[str] = None):
+    """
+    Validate a file item in a tar archive.
+
+    This function checks if the file item matches the expected size and hash.
+
+    :param file_item: The file item from the Hugging Face repository.
+    :type file_item: RepoFile
+    :param size: The expected size of the file.
+    :type size: int
+    :param hash_: The expected SHA-1 hash of the file.
+    :type hash_: str, optional
+    :param hash_lfs: The expected SHA-256 hash of the file if stored in LFS.
+    :type hash_lfs: str, optional
+    :return: True if the file item is valid, False otherwise.
+    :rtype: bool
+    """
+    # size not match
+    if (file_item.lfs and size != file_item.lfs.size) or \
+            (not file_item.lfs and size != file_item.size):
+        return False
+
+    # compare tar file hash
+    item_hashes = [file_item.blob_id]
+    if file_item.lfs:
+        item_hashes.append(file_item.lfs.sha256)
+    item_hashes = set(filter(bool, item_hashes))
+    cmp_hashes = [hash_, hash_lfs]
+    cmp_hashes = set(filter(bool, cmp_hashes))
+    return bool(cmp_hashes & item_hashes)
+
+
+def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                    idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                    idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                    hf_token: Optional[str] = None):
+    """
+    Validate a tar archive in a Hugging Face repository.
+
+    This function validates if the tar archive in the Hugging Face repository matches the expected size and hash.
+
+    :param repo_id: The ID of the Hugging Face repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the tar archive in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository, defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository, defaults to 'main'.
+    :type revision: str, optional
+    :param idx_repo_id: The ID of the repository where the index file is stored.
+    :type idx_repo_id: Optional[str], optional
+    :param idx_file_in_repo: The path to the index file in the repository.
+    :type idx_file_in_repo: Optional[str], optional
+    :param idx_repo_type: The type of the repository where the index file is stored.
+    :type idx_repo_type: Optional[RepoTypeTyping], optional
+    :param idx_revision: The revision of the repository where the index file is stored.
+    :type idx_revision: Optional[str], optional
+    :param hf_token: The Hugging Face token for authentication, defaults to None.
+    :type hf_token: Optional[str], optional
+    :raises EntryNotFoundError: If the specified entry is not found in the repository.
+    :raises IsADirectoryError: If the specified entry is a directory.
+    :return: True if the tar archive is valid, False otherwise.
+    :rtype: bool
+    """
+    hf_client = get_hf_client(hf_token)
+
+    items = list(hf_client.get_paths_info(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        paths=[archive_in_repo],
+        revision=revision,
+    ))
+    if len(items) == 0:
+        raise EntryNotFoundError(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} not found.')
+    elif not isinstance(items[0], RepoFile):
+        raise IsADirectoryError(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} is a directory, not a file.')
+    else:
+        item = items[0]
+
+    try:
+        index = hf_tar_get_index(
+            repo_id=repo_id,
+            archive_in_repo=archive_in_repo,
+            repo_type=repo_type,
+            revision=revision,
+
+            idx_repo_id=idx_repo_id,
+            idx_file_in_repo=idx_file_in_repo,
+            idx_repo_type=idx_repo_type,
+            idx_revision=idx_revision,
+
+            hf_token=hf_token,
+        )
+    except (EntryNotFoundError, RepositoryNotFoundError):
+        return False
+
+    return hf_tar_item_validate(
+        file_item=item,
+        size=index['filesize'],
+        hash_=index.get('hash'),
+        hash_lfs=index.get('hash_lfs'),
+    )

{hfutils-0.2.0 → hfutils-0.2.2/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.0
+Version: 0.2.2
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992

{hfutils-0.2.0 → hfutils-0.2.2}/hfutils.egg-info/SOURCES.txt

@@ -35,6 +35,7 @@ hfutils/index/__init__.py
 hfutils/index/fetch.py
 hfutils/index/hash.py
 hfutils/index/make.py
+hfutils/index/validate.py
 hfutils/operate/__init__.py
 hfutils/operate/base.py
 hfutils/operate/download.py