hfutils 0.1.2__tar.gz → 0.2.1__tar.gz

{hfutils-0.1.2/hfutils.egg-info → hfutils-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.1.2
+Version: 0.2.1
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -21,10 +21,11 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.20
+Requires-Dist: huggingface_hub>=0.22
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
+Requires-Dist: tzlocal
 Provides-Extra: rar
 Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: test
@@ -229,6 +230,14 @@ hfutils download -h
 
 ```
 
+## List Files in Repository
+
+List files in repositories
+
+```shell
+hfutils ls -r your/repository -o /your/local/file -d subdir/in/repo
+```
+
 ## Supported Formats
 
 By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.

{hfutils-0.1.2 → hfutils-0.2.1}/README.md

@@ -156,6 +156,14 @@ hfutils download -h
 
 ```
 
+## List Files in Repository
+
+List files in repositories
+
+```shell
+hfutils ls -r your/repository -o /your/local/file -d subdir/in/repo
+```
+
 ## Supported Formats
 
 By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.

{hfutils-0.1.2 → hfutils-0.2.1}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.1.2'
+__VERSION__ = '0.2.1'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.1.2 → hfutils-0.2.1}/hfutils/entry/cli.py

@@ -1,10 +1,12 @@
 from .dispatch import hfutilcli
 from .download import _add_download_subcommand
+from .ls import _add_ls_subcommand
 from .upload import _add_upload_subcommand
 
 _DECORATORS = [
     _add_download_subcommand,
     _add_upload_subcommand,
+    _add_ls_subcommand,
 ]
 
 cli = hfutilcli

hfutils-0.2.1/hfutils/entry/ls.py

@@ -0,0 +1,189 @@
+import mimetypes
+import os.path
+from enum import Enum, unique
+from typing import Union, List
+
+import click
+import tzlocal
+from huggingface_hub.hf_api import RepoFolder, RepoFile
+
+from .base import CONTEXT_SETTINGS
+from ..operate.base import REPO_TYPES, get_hf_client
+
+mimetypes.add_type('image/webp', '.webp')
+
+
+@unique
+class ListItemType(Enum):
+    """
+    Enum class representing different types of list items.
+    """
+
+    FILE = 0x1
+    FOLDER = 0x2
+    IMAGE = 0x3
+    ARCHIVE = 0x4
+    MODEL = 0x5
+    DATA = 0x6
+
+    @property
+    def render_color(self):
+        """
+        Get the render color based on the item type.
+
+        :return: The render color for the item type.
+        :rtype: str
+        """
+        if self == self.FILE:
+            return None
+        elif self == self.FOLDER:
+            return 'blue'
+        elif self == self.IMAGE:
+            return 'magenta'
+        elif self == self.ARCHIVE:
+            return 'red'
+        elif self == self.MODEL:
+            return 'green'
+        elif self == self.DATA:
+            return 'yellow'
+        else:
+            raise ValueError(f'Unknown type - {self!r}')  # pragma: no cover
+
+
+class ListItem:
+    """
+    Class representing a list item.
+
+    :param item: The item object.
+    :type item: Union[RepoFolder, RepoFile]
+    :param base_dir: The base directory.
+    :type base_dir: str
+    """
+
+    def __init__(self, item: Union[RepoFolder, RepoFile], base_dir: str):
+        self.item = item
+        self.base_dir = base_dir
+        if isinstance(item, RepoFolder):
+            self.type = ListItemType.FOLDER
+        else:
+            mimetype, _ = mimetypes.guess_type(item.path)
+            _, ext = os.path.splitext(item.path)
+            self.type = ListItemType.FILE
+            if ext in {'.ckpt', '.pt', '.safetensors', '.onnx', '.model', '.h5', '.mlmodel',
+                       '.ftz', '.pb', '.pth', '.tflite'}:
+                self.type = ListItemType.MODEL
+            elif ext in {'.json', '.csv', '.tsv', '.arrow', '.bin', '.msgpack', '.npy', '.npz',
+                         '.parquet', '.pickle', '.pkl', '.wasm'}:
+                self.type = ListItemType.DATA
+            elif mimetype:
+                if 'image' in mimetype:
+                    self.type = ListItemType.IMAGE
+                elif mimetype.split('/', maxsplit=1)[-1] in {'zip', 'rar', 'x-tar', 'x-7z-compressed'}:
+                    self.type = ListItemType.ARCHIVE
+
+
+def _add_ls_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the 'ls' subcommand to the CLI.
+
+    :param cli: The Click CLI application.
+    :type cli: click.Group
+    :return: The modified Click CLI application.
+    :rtype: click.Group
+    """
+
+    @cli.command('ls', help='List files from HuggingFace repository.\n\n'
+                            'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-r', '--repository', 'repo_id', type=str, required=True,
+                  help='Repository to download from.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-d', '--directory', 'dir_in_repo', type=str, default=None,
+                  help='Directory in repository to download the full directory tree.')
+    @click.option('-R', '--revision', 'revision', type=str, default='main',
+                  help='Revision of repository.', show_default=True)
+    @click.option('-a', '--all', 'show_all', is_flag=True, type=bool, default=False,
+                  help='Show all files, including hidden files.', show_default=True)
+    @click.option('-l', '--list', 'show_detailed', is_flag=True, type=bool, default=False,
+                  help='Show detailed file information.', show_default=True)
+    def ls(repo_id: str, repo_type: str, dir_in_repo, revision: str, show_all: bool, show_detailed: bool):
+        """
+        List files from a HuggingFace repository.
+
+        :param repo_id: The identifier of the repository.
+        :type repo_id: str
+        :param repo_type: The type of the HuggingFace repository.
+        :type repo_type: str
+        :param dir_in_repo: The directory in the repository to list files from.
+        :type dir_in_repo: str
+        :param revision: The revision of the repository.
+        :type revision: str
+        :param show_all: Flag to indicate whether to show all files, including hidden files.
+        :type show_all: bool
+        :param show_detailed: Flag to indicate whether to show detailed file information.
+        :type show_detailed: bool
+        """
+        hf_client = get_hf_client()
+        items: List[ListItem] = []
+        for item in hf_client.list_repo_tree(
+                repo_id=repo_id,
+                repo_type=repo_type,
+                path_in_repo=dir_in_repo,
+                revision=revision,
+                recursive=False,
+                expand=show_detailed,
+        ):
+            relpath = os.path.relpath(item.path, dir_in_repo)
+            if not show_all and relpath.startswith('.'):
+                continue
+
+            item = ListItem(item, dir_in_repo)
+            items.append(item)
+            if not show_detailed:
+                print(click.style(
+                    os.path.relpath(item.item.path, item.base_dir),
+                    fg=item.type.render_color
+                ))
+
+        if show_detailed:
+            max_size_length = 0
+            max_commit_info_length = 0
+            for item in items:
+                if item.type == ListItemType.FOLDER:
+                    size_text = '-'
+                else:
+                    size_text = str(item.item.size)
+                max_size_length = max(max_size_length, len(size_text))
+
+                commit_text = (item.item.last_commit.title or '').splitlines(keepends=False)[0]
+                max_commit_info_length = max(max_commit_info_length, len(commit_text))
+
+            for item in items:
+                print('d' if item.type == ListItemType.FOLDER else '-', end='')
+                print('L' if item.type != ListItemType.FOLDER and item.item.lfs else '-', end='')
+
+                commit_text = (item.item.last_commit.title or '').splitlines(keepends=False)[0]
+                print(
+                    ' ' + ' ' * (max_commit_info_length - len(commit_text)) +
+                    click.style(commit_text, underline=True),
+                    end=''
+                )
+
+                if item.type == ListItemType.FOLDER:
+                    size_text = '-'
+                else:
+                    size_text = str(item.item.size)
+                size_text = ' ' * (max_size_length - len(size_text)) + size_text
+                print(' ' + size_text, end='')
+
+                print(' ' + item.item.last_commit.oid[:8], end='')
+                commit_time = item.item.last_commit.date.astimezone(tzlocal.get_localzone())
+                print(' ' + commit_time.strftime('%Y-%m-%d %H:%M %Z'), end='')
+
+                print(' ' + click.style(
+                    os.path.relpath(item.item.path, item.base_dir),
+                    fg=item.type.render_color
+                ))
+
+    return cli

hfutils-0.2.1/hfutils/index/__init__.py

@@ -0,0 +1,4 @@
+from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
+    ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch
+from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
+from .validate import hf_tar_item_validate, hf_tar_validate

hfutils-0.2.1/hfutils/index/fetch.py

@@ -0,0 +1,284 @@
+import json
+import os.path
+from typing import Optional, Dict, Union
+
+from huggingface_hub.file_download import http_get, hf_hub_url
+from huggingface_hub.utils import build_hf_headers
+
+from .hash import _f_sha256
+from ..operate.base import RepoTypeTyping, get_hf_client
+
+
+class ArchiveStandaloneFileIncompleteDownload(Exception):
+    """
+    Exception raised when a standalone file in an archive is incompletely downloaded.
+    """
+
+
+class ArchiveStandaloneFileHashNotMatch(Exception):
+    """
+    Exception raised when the hash of a standalone file in an archive does not match.
+    """
+
+
+def hf_tar_get_index(repo_id: str, archive_in_repo: str,
+                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                     idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                     idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                     hf_token: Optional[str] = None):
+    """
+    Get the index of a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: The index of the tar archive file.
+    :rtype: Dict
+    """
+    hf_client = get_hf_client(hf_token)
+    body, _ = os.path.splitext(archive_in_repo)
+    default_index_file = f'{body}.json'
+    with open(hf_client.hf_hub_download(
+            repo_id=idx_repo_id or repo_id,
+            repo_type=idx_repo_type or repo_type,
+            filename=idx_file_in_repo or default_index_file,
+            revision=idx_revision or revision,
+    ), 'r') as f:
+        return json.load(f)
+
+
+def hf_tar_list_files(repo_id: str, archive_in_repo: str,
+                      repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                      idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                      idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                      hf_token: Optional[str] = None):
+    """
+    List files inside a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: The list of files inside the tar archive.
+    :rtype: List[str]
+    """
+    index_data = hf_tar_get_index(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        repo_type=repo_type,
+        revision=revision,
+
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+
+        hf_token=hf_token,
+    )
+
+    return list(index_data['files'].keys())
+
+
+def hf_tar_file_exists(repo_id: str, archive_in_repo: str, file_in_archive: str,
+                       repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                       idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                       idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                       hf_token: Optional[str] = None):
+    """
+    Check if a file exists inside a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: True if the file exists, False otherwise.
+    :rtype: bool
+    """
+    index = hf_tar_get_index(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        repo_type=repo_type,
+        revision=revision,
+
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+
+        hf_token=hf_token,
+    )
+    files = _hf_files_process(index['files'])
+    return _n_path(file_in_archive) in files
+
+
+def _n_path(path):
+    """
+    Normalize a file path.
+
+    :param path: The file path to normalize.
+    :type path: str
+    :return: The normalized file path.
+    :rtype: str
+    """
+    return os.path.normpath(os.path.join('/', path))
+
+
+def _hf_files_process(files: Dict[str, dict]):
+    """
+    Normalize file paths in a dictionary of files.
+
+    :param files: The dictionary of files.
+    :type files: Dict[str, dict]
+    :return: The dictionary of files with normalized paths.
+    :rtype: Dict[str, dict]
+    """
+    return {_n_path(key): value for key, value in files.items()}
+
+
+def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: str, local_file: str,
+                         repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                         idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                         idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                         proxies: Optional[Dict] = None, user_agent: Union[Dict, str, None] = None,
+                         headers: Optional[Dict[str, str]] = None, endpoint: Optional[str] = None,
+                         hf_token: Optional[str] = None):
+    """
+    Download a file from a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param local_file: The path to save the downloaded file locally.
+    :type local_file: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param proxies: The proxies to be used for the HTTP request.
+    :type proxies: Dict, optional
+    :param user_agent: The user agent for the HTTP request.
+    :type user_agent: Union[Dict, str, None], optional
+    :param headers: The additional headers for the HTTP request.
+    :type headers: Dict[str, str], optional
+    :param endpoint: The Hugging Face API endpoint.
+    :type endpoint: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    """
+    index = hf_tar_get_index(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        repo_type=repo_type,
+        revision=revision,
+
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+
+        hf_token=hf_token,
+    )
+    files = _hf_files_process(index['files'])
+    if _n_path(file_in_archive) not in files:
+        raise FileNotFoundError(f'File {file_in_archive!r} not found '
+                                f'in {repo_type}s/{repo_id}@{revision}/{archive_in_repo}.')
+
+    info = files[_n_path(file_in_archive)]
+    url_to_download = hf_hub_url(repo_id, archive_in_repo, repo_type=repo_type, revision=revision, endpoint=endpoint)
+    headers = build_hf_headers(
+        token=hf_token,
+        library_name=None,
+        library_version=None,
+        user_agent=user_agent,
+        headers=headers,
+    )
+    start_bytes = info['offset']
+    end_bytes = info['offset'] + info['size'] - 1
+    headers['Range'] = f'bytes={start_bytes}-{end_bytes}'
+
+    if os.path.dirname(local_file):
+        os.makedirs(os.path.dirname(local_file), exist_ok=True)
+    try:
+        with open(local_file, 'wb') as f:
+            http_get(
+                url_to_download,
+                f,
+                proxies=proxies,
+                resume_size=0,
+                headers=headers,
+                expected_size=info['size'],
+                displayed_filename=file_in_archive,
+            )
+
+        if os.path.getsize(local_file) != info['size']:
+            raise ArchiveStandaloneFileIncompleteDownload(
+                f'Expected size is {info["size"]}, but actually {os.path.getsize(local_file)} downloaded.'
+            )
+
+        if info.get('sha256'):
+            _sha256 = _f_sha256(local_file)
+            if _sha256 != info['sha256']:
+                raise ArchiveStandaloneFileHashNotMatch(
+                    f'Expected hash is {info["sha256"]!r}, but actually {_sha256!r} found.'
+                )
+
+    except Exception:
+        if os.path.exists(local_file):
+            os.remove(local_file)
+        raise

hfutils-0.2.1/hfutils/index/hash.py

@@ -0,0 +1,22 @@
+from hashlib import sha256
+
+
+def _f_sha256(file, chunk_for_hash: int = 1 << 20):
+    """
+    Calculate the SHA-256 hash of a file.
+
+    :param file: The path to the file.
+    :type file: str
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :return: The SHA-256 hash of the file.
+    :rtype: str
+    """
+    file_sha = sha256()
+    with open(file, 'rb') as f:
+        while True:
+            data = f.read(chunk_for_hash)
+            if not data:
+                break
+            file_sha.update(data)
+    return file_sha.hexdigest()

hfutils-0.2.1/hfutils/index/make.py

@@ -0,0 +1,215 @@
+import json
+import logging
+import os
+import tarfile
+from hashlib import sha256, sha1
+from typing import Optional
+
+from .hash import _f_sha256
+from ..archive import archive_pack
+from ..operate import download_file_to_file, upload_file_to_file, upload_directory_as_directory
+from ..operate.base import RepoTypeTyping
+from ..utils import TemporaryDirectory, tqdm
+
+
+def tar_get_index_info(src_tar_file, chunk_for_hash: int = 1 << 20, with_hash: bool = True, silent: bool = False):
+    """
+    Get the index information of a tar archive file.
+
+    :param src_tar_file: The path to the source tar archive file.
+    :type src_tar_file: str
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param silent: Whether to suppress progress bars and logging messages, defaults to False.
+    :type silent: bool, optional
+    :return: The index information of the tar archive file.
+    :rtype: dict
+    """
+    filesize = os.path.getsize(src_tar_file)
+    sha_common = sha1()
+    sha_common.update(f'blob {filesize}\0'.encode('utf-8'))
+    sha_lfs = sha256()
+    logging.info(f'Calculating hash of {src_tar_file!r} ...')
+    with open(src_tar_file, 'rb') as f:
+        # make sure the big files will not cause OOM
+        while True:
+            data = f.read(chunk_for_hash)
+            if not data:
+                break
+            sha_common.update(data)
+            sha_lfs.update(data)
+
+    logging.info(f'Indexing tar file {src_tar_file!r} ...')
+    files = {}
+    with tarfile.open(src_tar_file, mode='r|') as tar:
+        for tarinfo in tqdm(tar, desc='Indexing tar file ...', silent=silent):
+            tarinfo: tarfile.TarInfo
+            if tarinfo.isreg():
+                info = {
+                    'offset': tarinfo.offset_data,
+                    'size': tarinfo.size,
+                }
+                if with_hash:
+                    with TemporaryDirectory() as td:
+                        tar.extract(tarinfo, td)
+                        dst_file = os.path.join(td, tarinfo.name)
+                        info['sha256'] = _f_sha256(dst_file)
+
+                files[tarinfo.name] = info
+
+    return {
+        'filesize': filesize,
+        'hash': sha_common.hexdigest(),
+        'hash_lfs': sha_lfs.hexdigest(),
+        'files': files,
+    }
+
+
+def tar_create_index(src_tar_file, dst_index_file: Optional[str] = None,
+                     chunk_for_hash: int = 1 << 20, with_hash: bool = True, silent: bool = False):
+    """
+    Create an index file for a tar archive file.
+
+    :param src_tar_file: The path to the source tar archive file.
+    :type src_tar_file: str
+    :param dst_index_file: The path to save the index file, defaults to None.
+    :type dst_index_file: str, optional
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param silent: Whether to suppress progress bars and logging messages, defaults to False.
+    :type silent: bool, optional
+    :return: The path to the created index file.
+    :rtype: str
+    """
+    body, _ = os.path.splitext(src_tar_file)
+    dst_index_file = dst_index_file or f'{body}.json'
+    with open(dst_index_file, 'w') as f:
+        json.dump(tar_get_index_info(src_tar_file, chunk_for_hash, with_hash, silent), f)
+    return dst_index_file
+
+
+def hf_tar_create_index(repo_id: str, archive_in_repo: str,
+                        repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                        idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                        idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                        chunk_for_hash: int = 1 << 20, with_hash: bool = True, skip_when_synced: bool = True,
+                        hf_token: Optional[str] = None, ):
+    """
+    Create an index file for a tar archive file in a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the tar archive file.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository, defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository, defaults to 'main'.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository, defaults to None.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to save the index file in the index repository, defaults to None.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository, defaults to None.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository, defaults to None.
+    :type idx_revision: str, optional
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param skip_when_synced: Skip syncing when index is ready, defaults to True.
+    :type skip_when_synced: bool
+    :param hf_token: The Hugging Face access token, defaults to None.
+    :type hf_token: str, optional
+    """
+    body, _ = os.path.splitext(archive_in_repo)
+    default_index_filename = f'{body}.json'
+
+    from .validate import hf_tar_validate
+    if skip_when_synced and hf_tar_validate(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            archive_in_repo=archive_in_repo,
+            revision=revision,
+
+            idx_repo_id=idx_repo_id or repo_id,
+            idx_repo_type=idx_repo_type or repo_type,
+            idx_file_in_repo=idx_file_in_repo or default_index_filename,
+            idx_revision=idx_revision or revision,
+
+            hf_token=hf_token,
+    ):
+        logging.info(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} already indexed, skipped.')
+        return
+
+    with TemporaryDirectory() as td:
+        local_tar_file = os.path.join(td, os.path.basename(archive_in_repo))
+        download_file_to_file(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            file_in_repo=archive_in_repo,
+            local_file=local_tar_file,
+            revision=revision,
+            hf_token=hf_token,
+        )
+        dst_index_file = tar_create_index(local_tar_file, chunk_for_hash=chunk_for_hash, with_hash=with_hash)
+
+        upload_file_to_file(
+            repo_id=idx_repo_id or repo_id,
+            repo_type=idx_repo_type or repo_type,
+            file_in_repo=idx_file_in_repo or default_index_filename,
+            local_file=dst_index_file,
+            revision=idx_revision or revision,
+            hf_token=hf_token,
+            message=f'Create index for {repo_type}s/{repo_id}@{revision}/{archive_in_repo}',
+        )
+
+
+def hf_tar_create_from_directory(
+        repo_id: str, archive_in_repo: str, local_directory: str,
+        repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+        chunk_for_hash: int = 1 << 20, with_hash: bool = True,
+        silent: bool = False, hf_token: Optional[str] = None):
+    """
+    Create a tar archive file from a local directory and upload it to a Hugging Face repository.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to save the tar archive file in the repository.
+    :type archive_in_repo: str
+    :param local_directory: The path to the local directory to be archived.
+    :type local_directory: str
+    :param repo_type: The type of the Hugging Face repository, defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository, defaults to 'main'.
+    :type revision: str, optional
+    :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).
+    :type chunk_for_hash: int, optional
+    :param with_hash: Whether to include file hashes in the index, defaults to True.
+    :type with_hash: bool, optional
+    :param silent: Whether to suppress progress bars and logging messages, defaults to False.
+    :type silent: bool, optional
+    :param hf_token: The Hugging Face access token, defaults to None.
+    :type hf_token: str, optional
+    """
+    _, ext = os.path.splitext(archive_in_repo)
+    with TemporaryDirectory() as td:
+        local_tar_file = os.path.join(td, archive_in_repo)
+        if os.path.dirname(local_tar_file):
+            os.makedirs(os.path.dirname(local_tar_file), exist_ok=True)
+        archive_pack('tar', local_directory, local_tar_file, silent=silent)
+        tar_create_index(local_tar_file, chunk_for_hash=chunk_for_hash, with_hash=with_hash, silent=silent)
+
+        upload_directory_as_directory(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            path_in_repo='.',
+            local_directory=td,
+            revision=revision,
+            hf_token=hf_token,
+            message=f'Create indexed tar {repo_type}s/{repo_id}@{revision}/{archive_in_repo}'
+        )

hfutils-0.2.1/hfutils/index/validate.py

@@ -0,0 +1,111 @@
+from typing import Optional
+
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
+
+from .fetch import hf_tar_get_index
+from ..operate.base import RepoTypeTyping, get_hf_client
+
+
+def hf_tar_item_validate(file_item: RepoFile, size: int, hash_: Optional[str] = None, hash_lfs: Optional[str] = None):
+    """
+    Validate a file item in a tar archive.
+
+    This function checks if the file item matches the expected size and hash.
+
+    :param file_item: The file item from the Hugging Face repository.
+    :type file_item: RepoFile
+    :param size: The expected size of the file.
+    :type size: int
+    :param hash_: The expected SHA-1 hash of the file.
+    :type hash_: str, optional
+    :param hash_lfs: The expected SHA-256 hash of the file if stored in LFS.
+    :type hash_lfs: str, optional
+    :return: True if the file item is valid, False otherwise.
+    :rtype: bool
+    """
+    # size not match
+    if (file_item.lfs and size != file_item.lfs.size) or \
+            (not file_item.lfs and size != file_item.size):
+        return False
+
+    # compare tar file hash
+    item_hashes = [file_item.blob_id]
+    if file_item.lfs:
+        item_hashes.append(file_item.lfs.sha256)
+    item_hashes = set(filter(bool, item_hashes))
+    cmp_hashes = [hash_, hash_lfs]
+    cmp_hashes = set(filter(bool, cmp_hashes))
+    return bool(cmp_hashes & item_hashes)
+
+
+def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                    idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                    idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                    hf_token: Optional[str] = None):
+    """
+    Validate a tar archive in a Hugging Face repository.
+
+    This function validates if the tar archive in the Hugging Face repository matches the expected size and hash.
+
+    :param repo_id: The ID of the Hugging Face repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the tar archive in the repository.
+    :type archive_in_repo: str
+    :param repo_type: The type of the Hugging Face repository, defaults to 'dataset'.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository, defaults to 'main'.
+    :type revision: str, optional
+    :param idx_repo_id: The ID of the repository where the index file is stored.
+    :type idx_repo_id: Optional[str], optional
+    :param idx_file_in_repo: The path to the index file in the repository.
+    :type idx_file_in_repo: Optional[str], optional
+    :param idx_repo_type: The type of the repository where the index file is stored.
+    :type idx_repo_type: Optional[RepoTypeTyping], optional
+    :param idx_revision: The revision of the repository where the index file is stored.
+    :type idx_revision: Optional[str], optional
+    :param hf_token: The Hugging Face token for authentication, defaults to None.
+    :type hf_token: Optional[str], optional
+    :raises EntryNotFoundError: If the specified entry is not found in the repository.
+    :raises IsADirectoryError: If the specified entry is a directory.
+    :return: True if the tar archive is valid, False otherwise.
+    :rtype: bool
+    """
+    hf_client = get_hf_client(hf_token)
+
+    items = list(hf_client.get_paths_info(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        paths=[archive_in_repo],
+        revision=revision,
+    ))
+    if len(items) == 0:
+        raise EntryNotFoundError(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} not found.')
+    elif not isinstance(items[0], RepoFile):
+        raise IsADirectoryError(f'Entry {repo_type}s/{repo_id}/{archive_in_repo} is a directory, not a file.')
+    else:
+        item = items[0]
+
+    try:
+        index = hf_tar_get_index(
+            repo_id=repo_id,
+            archive_in_repo=archive_in_repo,
+            repo_type=repo_type,
+            revision=revision,
+
+            idx_repo_id=idx_repo_id,
+            idx_file_in_repo=idx_file_in_repo,
+            idx_repo_type=idx_repo_type,
+            idx_revision=idx_revision,
+
+            hf_token=hf_token,
+        )
+    except (EntryNotFoundError, RepositoryNotFoundError):
+        return False
+
+    return hf_tar_item_validate(
+        file_item=item,
+        size=index['filesize'],
+        hash_=index.get('hash'),
+        hash_lfs=index.get('hash_lfs'),
+    )

{hfutils-0.1.2 → hfutils-0.2.1/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.1.2
+Version: 0.2.1
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -21,10 +21,11 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.20
+Requires-Dist: huggingface_hub>=0.22
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
+Requires-Dist: tzlocal
 Provides-Extra: rar
 Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: test
@@ -229,6 +230,14 @@ hfutils download -h
 
 ```
 
+## List Files in Repository
+
+List files in repositories
+
+```shell
+hfutils ls -r your/repository -o /your/local/file -d subdir/in/repo
+```
+
 ## Supported Formats
 
 By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.

{hfutils-0.1.2 → hfutils-0.2.1}/hfutils.egg-info/SOURCES.txt

@@ -29,7 +29,13 @@ hfutils/entry/base.py
 hfutils/entry/cli.py
 hfutils/entry/dispatch.py
 hfutils/entry/download.py
+hfutils/entry/ls.py
 hfutils/entry/upload.py
+hfutils/index/__init__.py
+hfutils/index/fetch.py
+hfutils/index/hash.py
+hfutils/index/make.py
+hfutils/index/validate.py
 hfutils/operate/__init__.py
 hfutils/operate/base.py
 hfutils/operate/download.py

{hfutils-0.1.2 → hfutils-0.2.1}/hfutils.egg-info/requires.txt

@@ -1,8 +1,9 @@
 hbutils>=0.9.0
-huggingface_hub>=0.20
+huggingface_hub>=0.22
 tqdm
 requests
 click>=7
+tzlocal
 
 [7z]
 py7zr

{hfutils-0.1.2 → hfutils-0.2.1}/requirements.txt

@@ -1,5 +1,6 @@
 hbutils>=0.9.0
-huggingface_hub>=0.20
+huggingface_hub>=0.22
 tqdm
 requests
 click>=7
+tzlocal