hfutils 0.3.9__tar.gz → 0.4.0__tar.gz

{hfutils-0.3.9/hfutils.egg-info → hfutils-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.9
+Version: 0.4.0
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.3.9'
+__VERSION__ = '0.4.0'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils/entry/cli.py

@@ -5,6 +5,7 @@ from .index import _add_index_subcommand
 from .ls import _add_ls_subcommand
 from .ls_repo import _add_ls_repo_subcommand
 from .rollback import _add_rollback_subcommand
+from .tree import _add_tree_subcommand
 from .upload import _add_upload_subcommand
 from .whoami import _add_whoami_subcommand
 
@@ -17,6 +18,7 @@ _DECORATORS = [
     _add_index_subcommand,
     _add_rollback_subcommand,
     _add_clone_subcommand,
+    _add_tree_subcommand,
 ]
 
 cli = hfutilcli

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils/entry/download.py

@@ -7,7 +7,7 @@ from huggingface_hub import configure_http_backend
 
 from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
 from ..operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory
-from ..operate.base import REPO_TYPES, RepoTypeTyping
+from ..operate.base import REPO_TYPES, RepoTypeTyping, _IGNORE_PATTERN_UNSET
 from ..utils import get_requests_session
 
 
@@ -55,12 +55,15 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                   help='Just check the file size when validating the downloaded files.', show_default=True)
     @click.option('--tmpdir', 'tmpdir', type=str, default=None,
                   help='Use custom temporary Directory.', show_default=True)
+    @click.option('--all', 'show_all', is_flag=True, type=bool, default=False,
+                  help='Show all files, including hidden files.', show_default=True)
     @command_wrap()
     def download(
             repo_id: str, repo_type: RepoTypeTyping,
             file_in_repo: Optional[str], archive_in_repo: Optional[str], dir_in_repo: Optional[str],
             output_path: str, revision: str, max_workers: int,
-            password: Optional[str], wildcard: Optional[str], soft_mode_when_check: bool, tmpdir: Optional[str]
+            password: Optional[str], wildcard: Optional[str], soft_mode_when_check: bool, tmpdir: Optional[str],
+            show_all: bool = False,
     ):
         """
         Download data from HuggingFace repositories.
@@ -135,6 +138,7 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                 silent=False,
                 max_workers=max_workers,
                 soft_mode_when_check=soft_mode_when_check,
+                ignore_patterns=_IGNORE_PATTERN_UNSET if not show_all else [],
             )
 
         else:

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils/entry/ls.py

@@ -1,6 +1,5 @@
-import mimetypes
 import os.path
-from enum import Enum, unique
+import os.path
 from typing import Union, List
 
 import click
@@ -10,46 +9,7 @@ from huggingface_hub.hf_api import RepoFolder, RepoFile
 
 from .base import CONTEXT_SETTINGS
 from ..operate.base import REPO_TYPES, get_hf_client
-from ..utils import get_requests_session
-
-mimetypes.add_type('image/webp', '.webp')
-
-
-@unique
-class ListItemType(Enum):
-    """
-    Enum class representing different types of list items.
-    """
-
-    FILE = 0x1
-    FOLDER = 0x2
-    IMAGE = 0x3
-    ARCHIVE = 0x4
-    MODEL = 0x5
-    DATA = 0x6
-
-    @property
-    def render_color(self):
-        """
-        Get the render color based on the item type.
-
-        :return: The render color for the item type.
-        :rtype: str
-        """
-        if self == self.FILE:
-            return None
-        elif self == self.FOLDER:
-            return 'blue'
-        elif self == self.IMAGE:
-            return 'magenta'
-        elif self == self.ARCHIVE:
-            return 'red'
-        elif self == self.MODEL:
-            return 'green'
-        elif self == self.DATA:
-            return 'yellow'
-        else:
-            raise ValueError(f'Unknown type - {self!r}')  # pragma: no cover
+from ..utils import get_requests_session, FileItemType, get_file_type
 
 
 class ListItem:
@@ -66,22 +26,9 @@ class ListItem:
         self.item = item
         self.base_dir = base_dir
         if isinstance(item, RepoFolder):
-            self.type = ListItemType.FOLDER
+            self.type = FileItemType.FOLDER
         else:
-            mimetype, _ = mimetypes.guess_type(item.path)
-            _, ext = os.path.splitext(item.path)
-            self.type = ListItemType.FILE
-            if ext in {'.ckpt', '.pt', '.safetensors', '.onnx', '.model', '.h5', '.mlmodel',
-                       '.ftz', '.pb', '.pth', '.tflite'}:
-                self.type = ListItemType.MODEL
-            elif ext in {'.json', '.csv', '.tsv', '.arrow', '.bin', '.msgpack', '.npy', '.npz',
-                         '.parquet', '.pickle', '.pkl', '.wasm'}:
-                self.type = ListItemType.DATA
-            elif mimetype:
-                if 'image' in mimetype:
-                    self.type = ListItemType.IMAGE
-                elif mimetype.split('/', maxsplit=1)[-1] in {'zip', 'rar', 'x-tar', 'x-7z-compressed'}:
-                    self.type = ListItemType.ARCHIVE
+            self.type = get_file_type(item.path)
 
 
 def _add_ls_subcommand(cli: click.Group) -> click.Group:
@@ -154,7 +101,7 @@ def _add_ls_subcommand(cli: click.Group) -> click.Group:
             max_size_length = 0
             max_commit_info_length = 0
             for item in items:
-                if item.type == ListItemType.FOLDER:
+                if item.type == FileItemType.FOLDER:
                     size_text = '-'
                 else:
                     size_text = str(item.item.size)
@@ -164,8 +111,8 @@ def _add_ls_subcommand(cli: click.Group) -> click.Group:
                 max_commit_info_length = max(max_commit_info_length, len(commit_text))
 
             for item in items:
-                print('d' if item.type == ListItemType.FOLDER else '-', end='')
-                print('L' if item.type != ListItemType.FOLDER and item.item.lfs else '-', end='')
+                print('d' if item.type == FileItemType.FOLDER else '-', end='')
+                print('L' if item.type != FileItemType.FOLDER and item.item.lfs else '-', end='')
 
                 commit_text = (item.item.last_commit.title or '').splitlines(keepends=False)[0]
                 print(
@@ -174,7 +121,7 @@ def _add_ls_subcommand(cli: click.Group) -> click.Group:
                     end=''
                 )
 
-                if item.type == ListItemType.FOLDER:
+                if item.type == FileItemType.FOLDER:
                     size_text = '-'
                 else:
                     size_text = str(item.item.size)

hfutils-0.4.0/hfutils/entry/tree.py

@@ -0,0 +1,239 @@
+"""
+This module provides functionality for listing and displaying files from a HuggingFace repository in a tree-like structure.
+
+It includes functions for parsing repository paths, retrieving file information, and formatting the output as a tree.
+The module also defines a CLI command for easy interaction with the tree functionality.
+
+Key components:
+
+- TreeItem: A dataclass representing an item (file or folder) in the tree structure.
+- _get_tree: Function to retrieve the tree structure of files in a HuggingFace repository.
+- _add_tree_subcommand: Function to add the 'tree' subcommand to a Click CLI group.
+
+Usage:
+This module is typically used as part of a larger CLI application for interacting with HuggingFace repositories.
+The 'tree' command can be used to visualize the structure of files in a repository.
+"""
+
+import dataclasses
+import os
+import re
+from typing import Optional, List, Union
+
+import click
+from hbutils.string import format_tree
+from huggingface_hub import configure_http_backend
+from huggingface_hub.hf_api import RepoFile
+from natsort import natsorted
+
+from .base import CONTEXT_SETTINGS
+from ..operate.base import REPO_TYPES, list_files_in_repository, RepoTypeTyping, get_hf_client
+from ..utils import get_requests_session, hf_normpath, get_file_type, hf_fs_path, FileItemType
+
+
+@dataclasses.dataclass
+class TreeItem:
+    """
+    Represents an item (file or folder) in the tree structure.
+
+    :param name: The name of the item.
+    :type name: str
+    :param type_: The type of the item (file or folder).
+    :type type_: FileItemType
+    :param children: List of child items if this is a folder.
+    :type children: Optional[List[TreeItem]]
+    :param exist: Whether the item exists in the repository.
+    :type exist: bool
+
+    :ivar name: The name of the item.
+    :ivar type_: The type of the item.
+    :ivar children: List of child items.
+    :ivar exist: Existence status of the item.
+    """
+
+    name: str
+    type_: FileItemType
+    children: Optional[List['TreeItem']]
+    exist: bool = True
+
+    def get_name(self):
+        """
+        Get the formatted name of the item for display.
+
+        :return: Formatted name string with color and strike-through if applicable.
+        :rtype: str
+        """
+        return click.style(
+            self.name,
+            fg=self.type_.render_color if self.exist else None,
+            strikethrough=not self.exist,
+        ) + ('' if self.exist else ' <NOT EXIST>')
+
+    def get_children(self):
+        """
+        Get the children of this item if it's a folder.
+
+        :return: List of child items if folder, empty list otherwise.
+        :rtype: List[TreeItem]
+        """
+        return self.children if self.type_ == FileItemType.FOLDER else []
+
+
+def _get_tree(repo_id: str, repo_type: RepoTypeTyping, dir_in_repo: str,
+              revision: Optional[str] = None, show_all: bool = False) -> TreeItem:
+    """
+    Retrieve the tree structure of files in a HuggingFace repository.
+
+    :param repo_id: The ID of the repository.
+    :type repo_id: str
+    :param repo_type: The type of the repository.
+    :type repo_type: RepoTypeTyping
+    :param dir_in_repo: The directory in the repository to start from.
+    :type dir_in_repo: str
+    :param revision: The revision of the repository to use.
+    :type revision: Optional[str]
+    :param show_all: Whether to show hidden files.
+    :type show_all: bool
+
+    :return: The root TreeItem representing the directory structure.
+    :rtype: TreeItem
+    """
+    root = {}
+    for filepath in list_files_in_repository(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            subdir=dir_in_repo,
+            revision=revision,
+            ignore_patterns=[],
+    ):
+        filename = hf_normpath(os.path.relpath(filepath, dir_in_repo))
+        segments = re.split(r'[\\/]+', filename)
+        if any(segment.startswith('.') and segment != '.' for segment in segments) and not show_all:
+            continue
+
+        current_node = root
+        for i, segment in enumerate(segments):
+            if segment not in current_node:
+                if i == (len(segments) - 1):
+                    current_node[segment] = get_file_type(segment)
+                else:
+                    current_node[segment] = {}
+            current_node = current_node[segment]
+
+    root_name = hf_fs_path(
+        repo_id=repo_id,
+        repo_type=repo_type,
+        filename=dir_in_repo,
+        revision=revision,
+    )
+
+    def _recursion(cur_node: Union[dict, FileItemType], parent_name: str, is_exist: bool = False):
+        if isinstance(cur_node, dict):
+            return TreeItem(
+                name=parent_name,
+                type_=FileItemType.FOLDER,
+                children=[
+                    _recursion(cur_node=value, parent_name=name, is_exist=is_exist)
+                    for name, value in natsorted(cur_node.items())
+                ],
+                exist=is_exist,
+            )
+        else:
+            return TreeItem(
+                name=parent_name,
+                type_=cur_node,
+                children=[],
+                exist=is_exist,
+            )
+
+    exist = True
+    if not root:
+        hf_client = get_hf_client()
+        paths = hf_client.get_paths_info(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            revision=revision,
+            paths=[dir_in_repo],
+        )
+        if len(paths) == 0:
+            exist = False
+        elif len(paths) == 1:
+            pathobj = paths[0]
+            if isinstance(pathobj, RepoFile):  # the subdir is a file
+                root = get_file_type(dir_in_repo)
+        else:
+            assert len(paths) == 1, \
+                f'Multiple path {dir_in_repo!r} found in repo {root_name!r}, ' \
+                f'this must be caused by HuggingFace API.'  # pragma: no cover
+
+    return _recursion(
+        cur_node=root,
+        parent_name=root_name,
+        is_exist=exist,
+    )
+
+
+def _add_tree_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the 'tree' subcommand to a Click CLI group.
+
+    This function defines a new 'tree' command that lists files from a HuggingFace repository
+    in a tree-like structure.
+
+    :param cli: The Click CLI group to add the command to.
+    :type cli: click.Group
+
+    :return: The modified CLI group with the 'tree' command added.
+    :rtype: click.Group
+
+    Usage:
+        This function is typically called when setting up a CLI application:
+
+        cli = click.Group()
+        cli = _add_tree_subcommand(cli)
+    """
+
+    @cli.command('tree', help='List files as a tree from HuggingFace repository.\n\n'
+                              'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-r', '--repository', 'repo_id', type=str, required=True,
+                  help='Repository to download from.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-d', '--directory', 'dir_in_repo', type=str, default=None,
+                  help='Directory in repository to download the full directory tree.')
+    @click.option('-R', '--revision', 'revision', type=str, default='main',
+                  help='Revision of repository.', show_default=True)
+    @click.option('-a', '--all', 'show_all', is_flag=True, type=bool, default=False,
+                  help='Show all files, including hidden files.', show_default=True)
+    def tree(repo_id: str, repo_type: RepoTypeTyping, dir_in_repo, revision: str, show_all: bool):
+        """
+        List files as a tree from a HuggingFace repository in a tree-like structure.
+
+        :param repo_id: The ID of the repository.
+        :type repo_id: str
+        :param repo_type: The type of the repository.
+        :type repo_type: RepoTypeTyping
+        :param dir_in_repo: The directory in the repository to start from.
+        :type dir_in_repo: str
+        :param revision: The revision of the repository to use.
+        :type revision: str
+        :param show_all: Whether to show hidden files.
+        :type show_all: bool
+        """
+        configure_http_backend(get_requests_session)
+
+        _tree = _get_tree(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            dir_in_repo=dir_in_repo or '.',
+            revision=revision,
+            show_all=show_all,
+        )
+        print(format_tree(
+            _tree,
+            format_node=TreeItem.get_name,
+            get_children=TreeItem.get_children,
+        ))
+
+    return cli

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils/utils/__init__.py

@@ -1,4 +1,6 @@
+from .archive import is_archive_or_compressed
 from .binary import is_binary_file
+from .data import is_data_file
 from .download import download_file
 from .logging import ColoredFormatter
 from .number import number_to_tag
@@ -6,4 +8,5 @@ from .path import hf_normpath, hf_fs_path, parse_hf_fs_path, HfFileSystemPath
 from .session import TimeoutHTTPAdapter, get_requests_session, get_random_ua
 from .temp import TemporaryDirectory
 from .tqdm_ import tqdm
+from .type_ import FileItemType, get_file_type
 from .walk import walk_files

hfutils-0.4.0/hfutils/utils/archive.py

@@ -0,0 +1,157 @@
+"""
+This module provides functionality for identifying archive and compressed files based on their filenames.
+
+It includes a comprehensive list of known archive and compressed file extensions, as well as patterns for
+identifying split archives and other generic compressed file formats. The main
+function :func:`is_archive_or_compressed` can be used to determine if a given filename
+represents an archive or compressed file.
+
+The module is useful for file handling operations where it's necessary to distinguish between regular files
+and archives or compressed files.
+"""
+
+import os.path
+import re
+from typing import Union
+
+_ARCHIVE_KNOWN_EXTS = {
+    '.tar',  # Tape Archive
+    '.rar',  # Roshal Archive
+    '.rar5',  # RAR version 5
+    '.zip',  # ZIP archive
+    '.7z',  # 7-Zip archive
+    '.7zip',  # Alternative extension for 7-Zip
+    '.gz',  # Gzip compressed file
+    '.bz2',  # Bzip2 compressed file
+    '.xz',  # XZ compressed file
+    '.ace',  # ACE archive
+    '.lz',  # Lzip compressed file
+    '.lzma',  # LZMA compressed file
+    '.z',  # Compress (Unix) file
+    '.cab',  # Microsoft Cabinet file
+    '.arj',  # ARJ archive
+    '.iso',  # ISO disk image
+    '.lzh',  # LZH archive
+    '.sit',  # StuffIt archive
+    '.sitx',  # StuffIt X archive
+    '.sea',  # Self-Extracting Archive
+    '.alz',  # ALZip archive
+    '.egg',  # Python Egg
+    '.whl',  # Python Wheel
+    '.deb',  # Debian package
+    '.rpm',  # Red Hat Package Manager
+    '.pkg',  # macOS Installer Package
+    '.dmg',  # Apple Disk Image
+    '.msi',  # Microsoft Installer
+    '.tgz',  # Gzipped tar archive
+    '.tbz2',  # Bzip2 compressed tar archive
+    '.lzw',  # LZW compressed file
+    '.rz',  # RZip compressed file
+    '.lzo',  # Lempel-Ziv-Oberhumer compressed file
+    '.zst',  # Zstandard compressed file
+    '.tar.gz',  # Gzipped tar archive
+    '.tar.bz2',  # Bzip2 compressed tar archive
+    '.tar.xz',  # XZ compressed tar archive
+    '.tar.lz',  # Lzip compressed tar archive
+    '.tar.lzma',  # LZMA compressed tar archive
+    '.zipx',  # Extended ZIP archive
+    '.arc',  # ARC archive
+    '.ark',  # ARC archive (alternative extension)
+    '.lha',  # LHA archive
+    '.zoo',  # ZOO archive
+    '.gca',  # GCA archive
+    '.uc2',  # UC2 archive
+    '.uha',  # UHarc archive
+    '.war',  # Web Application Archive
+    '.ear',  # Enterprise Application aRchive
+    '.sar',  # SAR archive
+    '.jar',  # Java Archive
+    '.apk',  # Android Package Kit
+    '.xpi',  # XPInstall (Mozilla browser extension)
+    '.snap',  # Snap package (Ubuntu)
+    '.appimage',  # AppImage package
+    '.squashfs',  # Squashfs filesystem
+    '.cpio',  # CPIO archive
+    '.shar',  # Shell archive
+    '.lbr',  # LBR archive
+    '.mar',  # Mozilla Archive
+    '.sbx',  # Sandbox file
+    '.qcow2',  # QEMU Copy On Write 2
+    '.vdi',  # VirtualBox Disk Image
+    '.vhd',  # Virtual Hard Disk
+    '.vmdk',  # Virtual Machine Disk
+    '.ova',  # Open Virtual Appliance
+    '.xar',  # eXtensible ARchive
+    '.mpq',  # MoPaQ archive (Blizzard games)
+}
+
+# Additional generic patterns
+_EXTERNAL_PATTERNS = [
+    r'\.tar(?:\.(?:gz|bz2|xz|lz|lzma|Z))?$',  # Tar and compressed tar archives
+    r'\.t\.(?:gz|bz2|xz|lz|lzma|Z)$',  # Compressed files with .t.XX extension
+    r'\.(?:zip|z)$',  # ZIP archives and Z compressed files
+    r'\.rar$',  # RAR archives
+    r'\.7z$',  # 7-Zip archives
+    r'\.(?:ar|a)$',  # AR archives
+    r'\.(?:lz|lzma?)$',  # LZ and LZMA compressed files
+    r'\.gz(?:ip)?$',  # Gzip compressed files
+    r'\.bz(?:ip)?2?$',  # Bzip and Bzip2 compressed files
+    r'\.(?:xz|lzh|lha)$',  # XZ, LZH, and LHA archives
+    r'\.(?:iso|img|dmg|pkg|msi)$',  # Disk images and installers
+    r'\.(?:deb|rpm|apk|ipa)$',  # Package formats
+]
+
+_ARCHIVE_SPLIT_PATTERNS = [
+    r'^(.+\.(?:zip|rar|7z|tar|tar\.gz|tar\.bz2|tar\.xz))\.\d+$',  # Split archives (e.g., .zip.001)
+    r'^(.+\.t)\.[a-z0-9]{2,4}$',  # Split archives with .t.XX extension
+    r'^(.+\.part\d+)\.rar$',  # Split RAR archives
+    r'^(.+)\.r\d{2}$',  # Old-style split RAR archives
+]
+
+
+def is_archive_or_compressed(filename: Union[str, os.PathLike]) -> bool:
+    """
+    Determine if the given filename represents an archive or compressed file.
+
+    This function checks the filename against a list of known archive and compressed file extensions,
+    as well as patterns for split archives and other generic compressed file formats.
+
+    :param filename: The name of the file to check. Can be a string or a path-like object.
+    :type filename: Union[str, os.PathLike]
+
+    :return: True if the filename represents an archive or compressed file, False otherwise.
+    :rtype: bool
+
+    :raises TypeError: If the filename is not a string or path-like object.
+
+    Usage:
+        >>> is_archive_or_compressed('example.zip')
+        True
+        >>> is_archive_or_compressed('document.txt')
+        False
+        >>> is_archive_or_compressed('archive.tar.gz')
+        True
+        >>> is_archive_or_compressed('split_archive.zip.001')
+        True
+    """
+    if not isinstance(filename, (str, os.PathLike)):
+        raise TypeError(f'Unknown file name type - {filename!r}')
+    filename = os.path.basename(os.path.normcase(str(filename)))
+
+    # Check for known extensions
+    for ext in _ARCHIVE_KNOWN_EXTS:
+        if filename.lower().endswith(ext):
+            return True
+
+    # Check for split archives
+    for pattern in _ARCHIVE_SPLIT_PATTERNS:
+        match = re.match(pattern, filename.lower())
+        if match:
+            return True
+
+    # Check for external patterns
+    for pattern in _EXTERNAL_PATTERNS:
+        if re.search(pattern, filename.lower()):
+            return True
+
+    return False

hfutils-0.4.0/hfutils/utils/data.py

@@ -0,0 +1,87 @@
+"""
+This module provides functionality for identifying data files based on their file extensions.
+
+It includes a comprehensive set of data file extensions and a function to check if a given
+filename corresponds to a known data file format. This can be useful in various data processing
+and file handling scenarios where it's necessary to distinguish data files from other types of files.
+"""
+
+import os
+from typing import Union
+
+_DATA_EXTS = {
+    '.json',  # JavaScript Object Notation
+    '.csv',  # Comma-Separated Values
+    '.tsv',  # Tab-Separated Values
+    '.arrow',  # Apache Arrow file format
+    '.feather',  # Feather file format (fast, language-agnostic columnar format)
+    '.parquet',  # Apache Parquet file format
+    '.avro',  # Apache Avro file format
+    '.orc',  # Optimized Row Columnar file format
+    '.npy',  # NumPy array file
+    '.npz',  # NumPy compressed archive file
+    '.hdf5',  # Hierarchical Data Format version 5
+    '.h5',  # Alternative extension for HDF5
+    '.mat',  # MATLAB file format
+    '.sav',  # SPSS data file
+    '.dta',  # Stata data file
+    '.sas7bdat',  # SAS data file
+    '.xpt',  # SAS transport file
+    '.xlsx',  # Microsoft Excel Open XML Spreadsheet
+    '.xls',  # Microsoft Excel Binary File Format
+    '.ods',  # OpenDocument Spreadsheet
+    '.db',  # Generic database file
+    '.sqlite',  # SQLite database file
+    '.mdb',  # Microsoft Access database file
+    '.accdb',  # Microsoft Access database file (newer version)
+    '.dbf',  # dBase database file
+    '.ftr',  # Feather file format (alternative extension)
+    '.geojson',  # GeoJSON file (for geographical data)
+    '.shp',  # Shapefile (for geographical data)
+    '.kml',  # Keyhole Markup Language (for geographical data)
+    '.gpx',  # GPS Exchange Format
+    '.nc',  # NetCDF (Network Common Data Form) file
+    '.grib',  # GRIdded Binary or General Regularly-distributed Information in Binary form
+    '.hdf',  # Hierarchical Data Format (older version)
+    '.zarr',  # Zarr array storage format
+    '.bin',  # Generic Binary File
+    '.pickle',  # Pickle dumped file
+    '.pkl',  # Shortcut of .pickle
+    '.wasm',  # WASM
+}
+
+
+def is_data_file(filename: Union[str, os.PathLike]) -> bool:
+    """
+    Determine if a given filename corresponds to a known data file format.
+
+    This function checks if the file extension of the provided filename matches
+    any of the known data file extensions defined in the `_DATA_EXTS` set.
+
+    :param filename: The name of the file to check. Can be a string or a path-like object.
+    :type filename: Union[str, os.PathLike]
+
+    :return: True if the file extension matches a known data file format, False otherwise.
+    :rtype: bool
+
+    :raises TypeError: If the provided filename is not a string or path-like object.
+
+    Usage:
+        >>> is_data_file('data.csv')
+        True
+        >>> is_data_file('script.py')
+        False
+        >>> is_data_file(Path('/path/to/data.json'))
+        True
+
+    .. note::
+        The function is case-insensitive and works with both file names and full paths.
+        It normalizes the filename and extracts only the extension for comparison.
+    """
+    if not isinstance(filename, (str, os.PathLike)):
+        raise TypeError(f'Unknown file name type - {filename!r}')
+
+    # Normalize the filename and get the extension
+    filename = os.path.basename(os.path.normcase(str(filename)))
+    _, ext = os.path.splitext(filename.lower())
+    return ext in _DATA_EXTS

hfutils-0.4.0/hfutils/utils/model.py

@@ -0,0 +1,140 @@
+"""
+This module provides functionality for identifying model files based on their extensions and naming patterns.
+
+It includes a comprehensive list of model file extensions, patterns for sharded model files, and specific patterns
+for Hugging Face model files. The main function, :func:`is_model_file`, determines whether a given filename corresponds
+to a model file based on these predefined patterns and extensions.
+
+This module can be useful in various scenarios, such as:
+
+- Automated model file detection in directories
+- Validation of uploaded files in machine learning platforms
+- Preprocessing steps in model loading pipelines
+
+Usage:
+    .. code:: python
+
+        from model_file_identifier import is_model_file
+
+        filename = "model.pt"
+        if is_model_file(filename):
+            print(f"{filename} is a model file")
+        else:
+            print(f"{filename} is not a model file")
+"""
+
+import os
+import re
+from typing import Union
+
+_MODEL_EXTS = {
+    '.ckpt',  # Checkpoint file
+    '.pt',  # PyTorch model file
+    '.pth',  # PyTorch model file (alternative extension)
+    '.safetensors',  # SafeTensors model file
+    '.onnx',  # Open Neural Network Exchange model file
+    '.model',  # Generic model file
+    '.h5',  # Hierarchical Data Format version 5
+    '.hdf5',  # Hierarchical Data Format version 5 (alternative extension)
+    '.mlmodel',  # Core ML model file
+    '.ftz',  # FastText model file
+    '.pb',  # Protocol Buffer file (often used for TensorFlow models)
+    '.tflite',  # TensorFlow Lite model file
+    '.pkl',  # Pickle file (often used for scikit-learn models)
+    '.joblib',  # Joblib file (often used for scikit-learn models)
+    '.bin',  # Binary file (generic)
+    '.meta',  # Meta file (often associated with TensorFlow checkpoints)
+    '.params',  # Parameters file (often used in MXNet)
+    '.pdparams',  # PaddlePaddle parameters file
+    '.pdmodel',  # PaddlePaddle model file
+    '.ot',  # OpenVINO model file
+    '.nnet',  # Neural network file
+    '.dnn',  # Deep neural network file
+    '.mar',  # MXNet Archive
+    '.tf',  # TensorFlow SavedModel file
+    '.keras',  # Keras model file
+    '.weights',  # Weights file (generic)
+    '.pmml',  # Predictive Model Markup Language file
+    '.gguf',  # GGUF (GPT-Generated Unified Format) file
+    '.ggml',  # GGML (GPT-Generated Model Language) file
+    '.q4_0',  # 4-bit quantized model (type 0)
+    '.q4_1',  # 4-bit quantized model (type 1)
+    '.q5_0',  # 5-bit quantized model (type 0)
+    '.q5_1',  # 5-bit quantized model (type 1)
+    '.q8_0',  # 8-bit quantized model
+    '.qnt',  # Quantized model (generic)
+    '.int8',  # 8-bit integer quantized model
+    '.fp16',  # 16-bit floating point model
+    '.bk',  # Backup file (often used for model checkpoints)
+    '.engine',  # TensorRT engine file
+    '.plan',  # TensorRT plan file
+    '.trt',  # TensorRT model file
+    '.torchscript',  # TorchScript model file
+    '.pdiparams',  # PaddlePaddle inference parameters file
+    '.pdopt',  # PaddlePaddle optimizer file
+    '.nb',  # Neural network binary file
+    '.mnn',  # MNN (Mobile Neural Network) model file
+    '.ncnn',  # NCNN model file
+    '.om',  # CANN (Compute Architecture for Neural Networks) offline model
+    '.rknn',  # Rockchip Neural Network model file
+    '.xmodel',  # Vitis AI model file
+    '.kmodel',  # Kendryte model file
+}
+
+_MODEL_SHARD_PATTERNS = [
+    r'.*-\d{5}-of-\d{5}',  # Pattern for sharded files like "model-00001-of-00005"
+    r'.*\.bin\.\d+',  # Pattern for binary shards like "model.bin.1"
+    r'.*\.part\.\d+',  # Pattern for part files like "model.part.0"
+    r'.*_part_\d+',  # Alternative pattern for part files like "model_part_0"
+    r'.*-shard\d+',  # Pattern for shard files like "model-shard1"
+]
+
+_HF_MODEL_PATTERNS = [
+    r'pytorch_model.*\.bin',  # Hugging Face PyTorch model file
+    r'tf_model.*\.h5',  # Hugging Face TensorFlow model file
+    r'model.*\.ckpt',  # Hugging Face checkpoint file
+    r'flax_model.*\.msgpack',  # Hugging Face Flax model file
+    r'.*\.safetensors',  # SafeTensors file (often used in Hugging Face models)
+]
+
+
+def is_model_file(filename: Union[str, os.PathLike]) -> bool:
+    """
+    Determine if a given filename corresponds to a model file.
+
+    This function checks if the provided filename matches any of the known model file extensions
+    or patterns, including sharded model files and Hugging Face specific patterns.
+
+    :param filename: The name of the file to check. Can be a full path or just the filename.
+    :type filename: Union[str, os.PathLike]
+
+    :return: True if the filename corresponds to a model file, False otherwise.
+    :rtype: bool
+
+    :raises TypeError: If the filename is not a string or os.PathLike object.
+
+    Usage:
+        >>> is_model_file("model.pt")
+        True
+        >>> is_model_file("data.csv")
+        False
+        >>> is_model_file("model-00001-of-00005")
+        True
+        >>> is_model_file("pytorch_model.bin")
+        True
+
+    .. note::
+        This function is case-insensitive and works with both file names and full paths.
+    """
+    if not isinstance(filename, (str, os.PathLike)):
+        raise TypeError(f'Unknown file name type - {filename!r}')
+    filename = os.path.basename(os.path.normcase(str(filename)))
+
+    if any(filename.lower().endswith(ext) for ext in _MODEL_EXTS):
+        return True
+    if any(re.match(pattern, filename.lower()) for pattern in _MODEL_SHARD_PATTERNS):
+        return True
+    if any(re.match(pattern, filename.lower()) for pattern in _HF_MODEL_PATTERNS):
+        return True
+
+    return False

hfutils-0.4.0/hfutils/utils/type_.py

@@ -0,0 +1,121 @@
+"""
+This module provides functionality for determining file types and managing list item types.
+
+It includes an enumeration class for different types of list items, and a function to determine
+the type of a given file. The module also adds support for the WebP image format.
+
+The module uses the ``mimetypes`` library for MIME type detection and imports custom functions
+for identifying archive, model, and data files.
+"""
+
+import mimetypes
+import os
+from enum import Enum, unique
+from typing import Union
+
+from .archive import is_archive_or_compressed
+from .data import is_data_file
+from .model import is_model_file
+
+mimetypes.add_type('image/webp', '.webp')
+
+
+@unique
+class FileItemType(Enum):
+    """
+    Enum class representing different types of list items.
+
+    This enumeration defines various file and folder types that can be encountered
+    in a file system or list of items. Each type is associated with a unique integer value.
+
+    Usage:
+        >>> item_type = FileItemType.FILE
+        >>> print(item_type)
+        FileItemType.FILE
+        >>> print(item_type.value)
+        1
+    """
+
+    FILE = 0x1
+    FOLDER = 0x2
+    IMAGE = 0x3
+    ARCHIVE = 0x4
+    MODEL = 0x5
+    DATA = 0x6
+
+    @property
+    def render_color(self):
+        """
+        Get the render color based on the item type.
+
+        This property returns a color string associated with each item type,
+        which can be used for rendering or display purposes.
+
+        :return: The render color for the item type.
+        :rtype: str or None
+
+        :raises ValueError: If an unknown item type is encountered.
+
+        Usage:
+            >>> item_type = FileItemType.FOLDER
+            >>> print(item_type.render_color)
+            blue
+        """
+        if self == FileItemType.FILE:
+            return None
+        elif self == FileItemType.FOLDER:
+            return 'blue'
+        elif self == FileItemType.IMAGE:
+            return 'magenta'
+        elif self == FileItemType.ARCHIVE:
+            return 'red'
+        elif self == FileItemType.MODEL:
+            return 'green'
+        elif self == FileItemType.DATA:
+            return 'yellow'
+        else:
+            raise ValueError(f'Unknown type - {self!r}')  # pragma: no cover
+
+
+def get_file_type(filename: Union[str, os.PathLike]) -> FileItemType:
+    """
+    Determine the type of a given file.
+
+    This function analyzes the provided filename and returns the corresponding FileItemType.
+    It uses various methods to determine the file type, including checking for archives,
+    model files, data files, and image files based on MIME types.
+
+    :param filename: The name or path of the file to analyze.
+    :type filename: Union[str, os.PathLike]
+
+    :return: The determined FileItemType for the given file.
+    :rtype: FileItemType
+
+    :raises TypeError: If the provided filename is not a string or PathLike object.
+
+    Usage:
+        >>> file_type = get_file_type('image.jpg')
+        >>> print(file_type)
+        FileItemType.IMAGE
+
+        >>> file_type = get_file_type('data.csv')
+        >>> print(file_type)
+        FileItemType.DATA
+    """
+    if not isinstance(filename, (str, os.PathLike)):
+        raise TypeError(f'Unknown file name type - {filename!r}')
+    filename = os.path.basename(os.path.normcase(str(filename)))
+
+    mimetype, _ = mimetypes.guess_type(filename)
+    type_ = FileItemType.FILE
+    if is_archive_or_compressed(filename):
+        type_ = FileItemType.ARCHIVE
+    elif is_model_file(filename):
+        type_ = FileItemType.MODEL
+    elif is_data_file(filename):
+        type_ = FileItemType.DATA
+    elif mimetype:
+        if 'image' in mimetype:
+            type_ = FileItemType.IMAGE
+
+    return type_

{hfutils-0.3.9 → hfutils-0.4.0/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.9
+Version: 0.4.0
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992

{hfutils-0.3.9 → hfutils-0.4.0}/hfutils.egg-info/SOURCES.txt

@@ -37,6 +37,7 @@ hfutils/entry/index.py
 hfutils/entry/ls.py
 hfutils/entry/ls_repo.py
 hfutils/entry/rollback.py
+hfutils/entry/tree.py
 hfutils/entry/upload.py
 hfutils/entry/whoami.py
 hfutils/index/__init__.py
@@ -55,13 +56,17 @@ hfutils/repository/base.py
 hfutils/repository/clone.py
 hfutils/repository/rollback.py
 hfutils/utils/__init__.py
+hfutils/utils/archive.py
 hfutils/utils/binary.py
+hfutils/utils/data.py
 hfutils/utils/download.py
 hfutils/utils/irregular_repo.json
 hfutils/utils/logging.py
+hfutils/utils/model.py
 hfutils/utils/number.py
 hfutils/utils/path.py
 hfutils/utils/session.py
 hfutils/utils/temp.py
 hfutils/utils/tqdm_.py
+hfutils/utils/type_.py
 hfutils/utils/walk.py