hfutils 0.2.8__tar.gz → 0.2.9__tar.gz

{hfutils-0.2.8/hfutils.egg-info → hfutils-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.8
+Version: 0.2.9
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,6 +30,8 @@ Requires-Dist: natsort
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: transfer
+Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -50,10 +52,8 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: transfer
-Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -71,8 +71,8 @@ Requires-Dist: testtools>=2; extra == "test"
 Requires-Dist: where>=1.0.2; extra == "test"
 Requires-Dist: responses>=0.20.0; extra == "test"
 Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
 
 # hfutils
 

hfutils-0.2.9/hfutils/cache/__init__.py

@@ -0,0 +1,7 @@
+"""
+Overview:
+    Huggingface local cache management.
+
+    Can quickly manage the local cache and make the disk space under control.
+"""
+from .delete import delete_detached_cache, delete_cache

hfutils-0.2.9/hfutils/cache/delete.py

@@ -0,0 +1,151 @@
+import logging
+from functools import partial
+from typing import Optional, Callable
+
+from huggingface_hub import scan_cache_dir, CachedRepoInfo, CachedRevisionInfo, DeleteCacheStrategy
+
+
+def _collect_revisions(fn: Callable[[CachedRepoInfo, CachedRevisionInfo], bool], cache_dir: Optional[str] = None) \
+        -> DeleteCacheStrategy:
+    """
+    Collects revisions from the cache that match a specific condition defined by the function `fn`.
+
+    :param fn: A callable that takes a CachedRepoInfo and CachedRevisionInfo and returns a boolean.
+               If True, the revision matches the condition.
+    :type fn: Callable[[CachedRepoInfo, CachedRevisionInfo], bool]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    :return: A strategy for deleting the collected revisions.
+    :rtype: DeleteCacheStrategy
+    """
+    revision_hashes = set()
+    scan = scan_cache_dir(cache_dir=cache_dir)
+    for repo in scan.repos:
+        for revision in repo.revisions:
+            if fn(repo, revision):
+                revision_hashes.add(revision.commit_hash)
+    logging.info(f'Revisions to delete from huggingface cache: {sorted(revision_hashes)}')
+    return scan.delete_revisions(*revision_hashes)
+
+
+def _is_repo_match(repo: CachedRepoInfo, repo_id: Optional[str] = None, repo_type: Optional[str] = None) -> bool:
+    """
+    Checks if a repository matches the specified ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param repo_id: The ID of the repository to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The type of the repository to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the repository matches the conditions, False otherwise.
+    :rtype: bool
+    """
+    if repo_id and repo_type:
+        return repo.repo_id == repo_id and repo.repo_type == repo_type
+    elif repo_id:
+        return repo.repo_id == repo_id
+    elif repo_type:
+        return repo.repo_type == repo_type
+    else:
+        return True
+
+
+def _is_detached_revision(
+        repo: CachedRepoInfo, revision: CachedRevisionInfo,
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+) -> bool:
+    """
+    Determines if a revision is detached (no references) and matches the specified repository ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param revision: The revision information.
+    :type revision: CachedRevisionInfo
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the revision is detached and matches the repository conditions, False otherwise.
+    :rtype: bool
+    """
+    if len(revision.refs) == 0:
+        return _is_repo_match(repo, repo_id, repo_type)
+    else:
+        return False
+
+
+def delete_detached_cache(
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+        cache_dir: Optional[str] = None
+):
+    """
+    Deletes all detached revisions from the cache that match the specified repository ID and type.
+
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    """
+    # noinspection PyTypeChecker
+    strategy = _collect_revisions(
+        fn=partial(
+            _is_detached_revision,
+            repo_id=repo_id,
+            repo_type=repo_type,
+        ),
+        cache_dir=cache_dir,
+    )
+    logging.info(f'{strategy.expected_freed_size_str} space will be freed.')
+    strategy.execute()
+
+
+def _is_selected_revision(
+        repo: CachedRepoInfo, revision: CachedRevisionInfo,
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+) -> bool:
+    """
+    Determines if a revision matches the specified repository ID and type.
+
+    :param repo: The repository information.
+    :type repo: CachedRepoInfo
+    :param revision: The revision information.
+    :type revision: CachedRevisionInfo
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :return: True if the revision matches the repository conditions, False otherwise.
+    :rtype: bool
+    """
+    _ = repo, revision
+    return _is_repo_match(repo, repo_id, repo_type)
+
+
+def delete_cache(
+        repo_id: Optional[str] = None, repo_type: Optional[str] = None,
+        cache_dir: Optional[str] = None
+):
+    """
+    Deletes all revisions from the cache that match the specified repository ID and type.
+
+    :param repo_id: The repository ID to match. If None, matches any ID.
+    :type repo_id: Optional[str]
+    :param repo_type: The repository type to match. If None, matches any type.
+    :type repo_type: Optional[str]
+    :param cache_dir: The directory where the cache is stored. If None, uses the default directory.
+    :type cache_dir: Optional[str]
+    """
+    # noinspection PyTypeChecker
+    strategy = _collect_revisions(
+        fn=partial(
+            _is_selected_revision,
+            repo_id=repo_id,
+            repo_type=repo_type,
+        ),
+        cache_dir=cache_dir,
+    )
+    logging.info(f'{strategy.expected_freed_size_str} space will be freed.')
+    strategy.execute()

{hfutils-0.2.8 → hfutils-0.2.9}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.8'
+__VERSION__ = '0.2.9'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.8 → hfutils-0.2.9/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.8
+Version: 0.2.9
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -30,6 +30,8 @@ Requires-Dist: natsort
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Requires-Dist: setuptools<70; extra == "build"
+Provides-Extra: transfer
+Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -50,10 +52,8 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: transfer
-Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -71,8 +71,8 @@ Requires-Dist: testtools>=2; extra == "test"
 Requires-Dist: where>=1.0.2; extra == "test"
 Requires-Dist: responses>=0.20.0; extra == "test"
 Requires-Dist: natsort; extra == "test"
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
 
 # hfutils
 

{hfutils-0.2.8 → hfutils-0.2.9}/hfutils.egg-info/SOURCES.txt

@@ -23,6 +23,8 @@ hfutils/archive/rar.py
 hfutils/archive/sevenz.py
 hfutils/archive/tar.py
 hfutils/archive/zip.py
+hfutils/cache/__init__.py
+hfutils/cache/delete.py
 hfutils/config/__init__.py
 hfutils/config/meta.py
 hfutils/entry/__init__.py