hfutils 0.2.8__tar.gz → 0.2.10__tar.gz

{hfutils-0.2.8/hfutils.egg-info → hfutils-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.8
+Version: 0.2.10
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -27,9 +27,6 @@ Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -50,10 +47,15 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -71,8 +73,6 @@ Requires-Dist: testtools>=2; extra == "test"
 Requires-Dist: where>=1.0.2; extra == "test"
 Requires-Dist: responses>=0.20.0; extra == "test"
 Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

hfutils-0.2.10/hfutils/cache/__init__.py

@@ -0,0 +1,7 @@
+"""
+Overview:
+    Huggingface local cache management.
+
+    Can quickly manage the local cache and make the disk space under control.
+"""
+from .delete import delete_detached_cache, delete_cache

hfutils-0.2.10/hfutils/cache/delete.py

@@ -0,0 +1,157 @@
+import logging
+from functools import partial
+from typing import Optional, Callable
+
+from huggingface_hub import scan_cache_dir, CachedRepoInfo, CachedRevisionInfo, DeleteCacheStrategy, CacheNotFound
+
+
+def _collect_revisions(fn: Callable[[CachedRepoInfo, CachedRevisionInfo], bool], cache_dir: Optional[str] = None) \
+        -> DeleteCacheStrategy:
+    """
+    Collects revisions from the cache that match a specific condition defined by the function `fn`.
+
+    :param fn: A callable that takes a CachedRepoInfo and CachedRevisionInfo and returns a boolean.
+               If True, the revision matches the condition.
+    :type fn: Callable[[CachedRepoInfo, CachedRevisionInfo], bool]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    :return: A strategy for deleting the collected revisions.
+    :rtype: DeleteCacheStrategy
+    """
+    revision_hashes = set()
+    scan = scan_cache_dir(cache_dir=cache_dir)
+    for repo in scan.repos:
+        for revision in repo.revisions:
+            if fn(repo, revision):
+                revision_hashes.add(revision.commit_hash)
+    logging.info(f'Revisions to delete from huggingface cache: {sorted(revision_hashes)}')
+    return scan.delete_revisions(*revision_hashes)
+
+
+def _is_repo_match(repo: CachedRepoInfo, repo_id: Optional[str] = None, repo_type: Optional[str] = None) -> bool:
+    """
+    Checks if a repository matches the specified ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param repo_id: The ID of the repository to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The type of the repository to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the repository matches the conditions, False otherwise.
+    :rtype: bool
+    """
+    if repo_id and repo_type:
+        return repo.repo_id == repo_id and repo.repo_type == repo_type
+    elif repo_id:
+        return repo.repo_id == repo_id
+    elif repo_type:
+        return repo.repo_type == repo_type
+    else:
+        return True
+
+
+def _is_detached_revision(
+        repo: CachedRepoInfo, revision: CachedRevisionInfo,
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+) -> bool:
+    """
+    Determines if a revision is detached (no references) and matches the specified repository ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param revision: The revision information.
+    :type revision: CachedRevisionInfo
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the revision is detached and matches the repository conditions, False otherwise.
+    :rtype: bool
+    """
+    if len(revision.refs) == 0:
+        return _is_repo_match(repo, repo_id, repo_type)
+    else:
+        return False
+
+
+def delete_detached_cache(
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+        cache_dir: Optional[str] = None
+):
+    """
+    Deletes all detached revisions from the cache that match the specified repository ID and type.
+
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    """
+    try:
+        # noinspection PyTypeChecker
+        strategy = _collect_revisions(
+            fn=partial(
+                _is_detached_revision,
+                repo_id=repo_id,
+                repo_type=repo_type,
+            ),
+            cache_dir=cache_dir,
+        )
+        logging.info(f'{strategy.expected_freed_size_str} space will be freed.')
+        strategy.execute()
+    except CacheNotFound:
+        logging.info('No cache file found.')
+
+
+def _is_selected_revision(
+        repo: CachedRepoInfo, revision: CachedRevisionInfo,
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+) -> bool:
+    """
+    Determines if a revision matches the specified repository ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param revision: The revision information.
+    :type revision: CachedRevisionInfo
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the revision matches the repository conditions, False otherwise.
+    :rtype: bool
+    """
+    _ = repo, revision
+    return _is_repo_match(repo, repo_id, repo_type)
+
+
+def delete_cache(
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+        cache_dir: Optional[str] = None
+):
+    """
+    Deletes all revisions from the cache that match the specified repository ID and type.
+
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    """
+    try:
+        # noinspection PyTypeChecker
+        strategy = _collect_revisions(
+            fn=partial(
+                _is_selected_revision,
+                repo_id=repo_id,
+                repo_type=repo_type,
+            ),
+            cache_dir=cache_dir,
+        )
+        logging.info(f'{strategy.expected_freed_size_str} space will be freed.')
+        strategy.execute()
+    except CacheNotFound:
+        logging.info('No cache file found.')

{hfutils-0.2.8 → hfutils-0.2.10}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.8'
+__VERSION__ = '0.2.10'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.8 → hfutils-0.2.10}/hfutils/entry/cli.py

@@ -1,5 +1,6 @@
 from .dispatch import hfutilcli
 from .download import _add_download_subcommand
+from .index import _add_index_subcommand
 from .ls import _add_ls_subcommand
 from .ls_repo import _add_ls_repo_subcommand
 from .upload import _add_upload_subcommand
@@ -11,6 +12,7 @@ _DECORATORS = [
     _add_ls_subcommand,
     _add_whoami_subcommand,
     _add_ls_repo_subcommand,
+    _add_index_subcommand,
 ]
 
 cli = hfutilcli

hfutils-0.2.10/hfutils/entry/index.py

@@ -0,0 +1,197 @@
+"""
+This module provides functionality for creating and managing index repositories for tar archives stored in Hugging Face repositories.
+
+The main features of this module include:
+- Adding an 'index' subcommand to a Click CLI application
+- Creating index files for tar archives
+- Uploading index files to a specified Hugging Face repository
+- Managing the upload process with a minimum interval between uploads
+
+The module uses the Hugging Face file system and client for interacting with repositories.
+"""
+
+import logging
+import os.path
+import shutil
+import time
+from typing import Optional
+
+import click
+from hbutils.string import plural_word
+
+from .base import CONTEXT_SETTINGS
+from ..cache import delete_detached_cache
+from ..index import hf_tar_validate, tar_create_index
+from ..operate import get_hf_fs, download_file_to_file, upload_directory_as_directory
+from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
+from ..utils import tqdm, hf_fs_path, parse_hf_fs_path, TemporaryDirectory, hf_normpath, ColoredFormatter
+
+
+def _add_index_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the 'index' subcommand to the CLI.
+
+    This function defines and adds the 'index' subcommand to the provided Click CLI application.
+    The subcommand is responsible for creating index files for tar archives in a Hugging Face repository
+    and uploading them to a specified index repository.
+
+    :param cli: The Click CLI application to which the 'index' subcommand will be added.
+    :type cli: click.Group
+
+    :return: The modified Click CLI application with the 'index' subcommand added.
+    :rtype: click.Group
+
+    Usage:
+        This function is typically called when setting up the CLI application, like this:
+        cli = click.Group()
+        cli = _add_index_subcommand(cli)
+    """
+
+    @cli.command('index', help='Make index repository for tar repository.\n\n'
+                               'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-r', '--repository', 'repo_id', type=str, required=True,
+                  help='Repository with tar archives inside.')
+    @click.option('-x', '--index_repository', 'idx_repo_id', type=str, default=None,
+                  help='Repository to create index files. Use the same repository as `-r` option when not given.',
+                  show_default=True)
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-R', '--revision', 'revision', type=str, default='main',
+                  help='Revision of repository.', show_default=True)
+    @click.option('--min_upload_interval', 'min_upload_interval', type=float, default=60,
+                  help='Min seconds for uploading to huggingface repository.', show_default=True)
+    def index(repo_id: str, idx_repo_id: Optional[str], repo_type: RepoTypeTyping, revision: str,
+              min_upload_interval: float):
+        """
+        Create index files for tar archives in a Hugging Face repository and upload them to a specified index repository.
+
+        This function sets up logging, creates the index repository if it doesn't exist, processes tar archives
+        to create index files, and manages the upload process of these index files to the specified repository.
+
+        :param repo_id: The ID of the repository containing tar archives.
+        :type repo_id: str
+        :param idx_repo_id: The ID of the repository where index files will be uploaded. If None, uses repo_id.
+        :type idx_repo_id: Optional[str]
+        :param repo_type: The type of the Hugging Face repository (e.g., 'dataset', 'model').
+        :type repo_type: RepoTypeTyping
+        :param revision: The revision of the repository to use.
+        :type revision: str
+        :param min_upload_interval: The minimum time interval (in seconds) between uploads to the Hugging Face repository.
+        :type min_upload_interval: float
+
+        :raises: Various exceptions may be raised during file system operations or API calls.
+
+        Usage:
+            This function is typically invoked through the CLI interface, like:
+            $ python script.py index -r my_repo -x my_index_repo -t dataset -R main --min_upload_interval 120
+        """
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(ColoredFormatter())
+        logger.addHandler(console_handler)
+
+        idx_repo_id = idx_repo_id or repo_id
+        delete_detached_cache(repo_id=repo_id, repo_type=repo_type)
+
+        hf_client = get_hf_client()
+        if not hf_client.repo_exists(repo_id=idx_repo_id, repo_type=repo_type):
+            logging.info(f'Creating repository {idx_repo_id!r}, type: {repo_type!r} ...')
+            hf_client.create_repo(repo_id=idx_repo_id, repo_type=repo_type)
+
+        hf_fs = get_hf_fs()
+        hf_tars = hf_fs.glob(hf_fs_path(repo_id=repo_id, repo_type=repo_type, filename='**/*.tar', revision=revision))
+        logging.info(f'{plural_word(len(hf_tars), "tar archive")} found in {repo_type}s/{repo_id} ...')
+        with TemporaryDirectory() as upload_dir:
+            idx_files = []
+            last_uploaded_at: Optional[float] = None
+
+            def _upload(force=False):
+                """
+                Upload index files to the Hugging Face repository.
+
+                This internal function manages the upload process of index files, respecting the minimum upload interval.
+
+                :param force: If True, upload regardless of the time since the last upload.
+                :type force: bool
+                """
+                nonlocal last_uploaded_at
+
+                if not idx_files:
+                    return
+                if not force and last_uploaded_at and last_uploaded_at + min_upload_interval > time.time():
+                    return
+
+                logging.info(f'Uploading index for {plural_word(len(idx_files), "tar archives")} ...')
+                upload_directory_as_directory(
+                    repo_id=idx_repo_id,
+                    repo_type=repo_type,
+                    revision=revision,
+                    local_directory=upload_dir,
+                    path_in_repo='.',
+                    message=f'Index for {plural_word(len(idx_files), "tar archives")} - '
+                            f'{", ".join(map(repr, idx_files))}',
+                )
+                last_uploaded_at = time.time()
+
+                _clean_dir()
+
+            def _clean_dir():
+                """
+                Clean the temporary upload directory.
+
+                This internal function removes all files and directories in the upload directory.
+                """
+                logging.info(f'Cleaning upload temporary directory {upload_dir!r} ...')
+                for item in os.listdir(upload_dir):
+                    dst_file = os.path.join(upload_dir, item)
+                    if os.path.isfile(dst_file):
+                        os.remove(dst_file)
+                    elif os.path.islink(dst_file):
+                        os.unlink(dst_file)
+                    elif os.path.isdir(dst_file):
+                        shutil.rmtree(dst_file)
+                idx_files.clear()
+                logging.info('Upload directory cleaned.')
+
+            for tar_path in tqdm(hf_tars, desc='Making Indices'):
+                tar_filename = parse_hf_fs_path(tar_path).filename
+                idx_filename = hf_normpath(os.path.splitext(tar_filename)[0] + '.json')
+                if not hf_tar_validate(
+                        repo_id=repo_id,
+                        repo_type=repo_type,
+                        archive_in_repo=tar_filename,
+                        idx_repo_id=idx_repo_id,
+                        revision=revision,
+                ):
+                    src_hf_path = hf_fs_path(repo_id=repo_id, repo_type=repo_type,
+                                             filename=tar_filename, revision=revision)
+                    dst_hf_path = hf_fs_path(repo_id=idx_repo_id, repo_type=repo_type,
+                                             filename=idx_filename, revision=revision)
+                    logging.info(f'Indexing {src_hf_path!r} --> {dst_hf_path!r} ...')
+                    with TemporaryDirectory() as td:
+                        local_tar_file = os.path.join(td, os.path.basename(tar_filename))
+                        local_idx_file = os.path.join(upload_dir, idx_filename)
+                        if os.path.dirname(local_idx_file):
+                            os.makedirs(os.path.dirname(local_idx_file), exist_ok=True)
+
+                        download_file_to_file(
+                            repo_id=repo_id,
+                            repo_type=repo_type,
+                            file_in_repo=tar_filename,
+                            local_file=local_tar_file,
+                            revision=revision,
+                        )
+                        logging.debug(f'Creating index file {local_tar_file!r} --> {local_idx_file!r} ...')
+                        tar_create_index(
+                            src_tar_file=local_tar_file,
+                            dst_index_file=local_idx_file,
+                        )
+                        idx_files.append(idx_filename)
+
+                _upload(force=False)
+
+            _upload(force=True)
+
+    return cli

{hfutils-0.2.8 → hfutils-0.2.10}/hfutils/utils/__init__.py

@@ -1,5 +1,6 @@
 from .binary import is_binary_file
 from .download import download_file
+from .logging import ColoredFormatter
 from .path import hf_normpath, hf_fs_path, parse_hf_fs_path, HfFileSystemPath
 from .temp import TemporaryDirectory
 from .tqdm_ import tqdm

hfutils-0.2.10/hfutils/utils/logging.py

@@ -0,0 +1,69 @@
+"""
+Overview:
+    This module provides functionality to enhance logging output with colors for different log levels.
+    It includes a `Colors` class defining ANSI escape sequences for various colors and styles,
+    and a `ColoredFormatter` class to format log messages with these colors based on their severity level.
+"""
+
+import logging
+
+
+class Colors:
+    """
+    A collection of ANSI escape sequences for terminal text coloring and styling.
+    These constants can be used to format strings with various colors and text styles.
+    """
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    BLACK = "\033[30m"
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    MAGENTA = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+    BRIGHT_BLACK = "\033[90m"
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_MAGENTA = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+
+class ColoredFormatter(logging.Formatter):
+    """
+    A logging formatter that applies colors to log messages based on their severity level.
+
+    The colors are defined in the `Colors` class and are applied to different parts of the log message
+    such as the timestamp, log level, logger name, and the message itself.
+    """
+    COLORS = {
+        'DEBUG': Colors.BLUE,
+        'INFO': Colors.GREEN,
+        'WARNING': Colors.YELLOW,
+        'ERROR': Colors.RED,
+        'CRITICAL': Colors.BOLD + Colors.RED,
+    }
+
+    def format(self, record):
+        """
+        Format the specified record as text, applying color based on the log level.
+
+        :param record: The log record to be formatted.
+        :type record: logging.LogRecord
+
+        :return: The formatted log message with appropriate colors.
+        :rtype: str
+        """
+        log_color = self.COLORS.get(record.levelname, Colors.RESET)
+        format_str = f"{Colors.BRIGHT_BLACK}[%(asctime)s]{Colors.RESET} "
+        format_str += f"{log_color}%(levelname)-8s{Colors.RESET} "
+        format_str += f"{Colors.CYAN}%(name)s{Colors.RESET} - "
+        format_str += f"{Colors.WHITE}%(message)s{Colors.RESET}"
+
+        formatter = logging.Formatter(format_str, datefmt="%Y-%m-%d %H:%M:%S")
+        return formatter.format(record)

{hfutils-0.2.8 → hfutils-0.2.10/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.8
+Version: 0.2.10
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -27,9 +27,6 @@ Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -50,10 +47,15 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -71,8 +73,6 @@ Requires-Dist: testtools>=2; extra == "test"
 Requires-Dist: where>=1.0.2; extra == "test"
 Requires-Dist: responses>=0.20.0; extra == "test"
 Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.2.8 → hfutils-0.2.10}/hfutils.egg-info/SOURCES.txt

@@ -23,6 +23,8 @@ hfutils/archive/rar.py
 hfutils/archive/sevenz.py
 hfutils/archive/tar.py
 hfutils/archive/zip.py
+hfutils/cache/__init__.py
+hfutils/cache/delete.py
 hfutils/config/__init__.py
 hfutils/config/meta.py
 hfutils/entry/__init__.py
@@ -30,6 +32,7 @@ hfutils/entry/base.py
 hfutils/entry/cli.py
 hfutils/entry/dispatch.py
 hfutils/entry/download.py
+hfutils/entry/index.py
 hfutils/entry/ls.py
 hfutils/entry/ls_repo.py
 hfutils/entry/upload.py
@@ -48,6 +51,7 @@ hfutils/utils/__init__.py
 hfutils/utils/binary.py
 hfutils/utils/download.py
 hfutils/utils/irregular_repo.json
+hfutils/utils/logging.py
 hfutils/utils/path.py
 hfutils/utils/temp.py
 hfutils/utils/tqdm_.py